Repository: apache/arrow-datafusion-comet Branch: main Commit: f635cad80ea8 Files: 2133 Total size: 19.8 MB Directory structure: gitextract_2d7o17rp/ ├── .asf.yaml ├── .claude/ │ └── skills/ │ ├── audit-comet-expression/ │ │ └── SKILL.md │ └── review-comet-pr/ │ └── SKILL.md ├── .dockerignore ├── .github/ │ ├── ISSUE_TEMPLATE/ │ │ ├── bug_report.yml │ │ └── feature_request.yml │ ├── actions/ │ │ ├── java-test/ │ │ │ └── action.yaml │ │ ├── rust-test/ │ │ │ └── action.yaml │ │ ├── setup-builder/ │ │ │ └── action.yaml │ │ ├── setup-iceberg-builder/ │ │ │ └── action.yaml │ │ ├── setup-macos-builder/ │ │ │ └── action.yaml │ │ └── setup-spark-builder/ │ │ └── action.yaml │ ├── dependabot.yml │ ├── pull_request_template.md │ └── workflows/ │ ├── codeql.yml │ ├── docker-publish.yml │ ├── docs.yaml │ ├── iceberg_spark_test.yml │ ├── label_new_issues.yml │ ├── miri.yml │ ├── pr_benchmark_check.yml │ ├── pr_build_linux.yml │ ├── pr_build_macos.yml │ ├── pr_markdown_format.yml │ ├── pr_missing_suites.yml │ ├── pr_rat_check.yml │ ├── pr_title_check.yml │ ├── spark_sql_test.yml │ ├── spark_sql_test_native_iceberg_compat.yml │ ├── stale.yml │ ├── take.yml │ └── validate_workflows.yml ├── .gitignore ├── .mvn/ │ └── wrapper/ │ ├── maven-wrapper.jar │ └── maven-wrapper.properties ├── .scalafix.conf ├── CHANGELOG.md ├── LICENSE.txt ├── Makefile ├── NOTICE.txt ├── README.md ├── benchmarks/ │ ├── Dockerfile │ ├── README.md │ ├── pyspark/ │ │ ├── README.md │ │ ├── benchmarks/ │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ └── shuffle.py │ │ ├── generate_data.py │ │ ├── run_all_benchmarks.sh │ │ └── run_benchmark.py │ └── tpc/ │ ├── .gitignore │ ├── README.md │ ├── create-iceberg-tables.py │ ├── drop-caches.sh │ ├── engines/ │ │ ├── comet-hashjoin.toml │ │ ├── comet-iceberg-hashjoin.toml │ │ ├── comet-iceberg.toml │ │ ├── comet.toml │ │ ├── gluten.toml │ │ └── spark.toml │ ├── generate-comparison.py │ ├── infra/ │ │ └── docker/ │ │ ├── Dockerfile │ │ ├── Dockerfile.build-comet │ │ ├── docker-compose-laptop.yml │ │ └── docker-compose.yml │ ├── queries/ │ │ ├── tpcds/ │ │ │ ├── q1.sql │ │ │ ├── q10.sql │ │ │ ├── q11.sql │ │ │ ├── q12.sql │ │ │ ├── q13.sql │ │ │ ├── q14.sql │ │ │ ├── q15.sql │ │ │ ├── q16.sql │ │ │ ├── q17.sql │ │ │ ├── q18.sql │ │ │ ├── q19.sql │ │ │ ├── q2.sql │ │ │ ├── q20.sql │ │ │ ├── q21.sql │ │ │ ├── q22.sql │ │ │ ├── q23.sql │ │ │ ├── q24.sql │ │ │ ├── q25.sql │ │ │ ├── q26.sql │ │ │ ├── q27.sql │ │ │ ├── q28.sql │ │ │ ├── q29.sql │ │ │ ├── q3.sql │ │ │ ├── q30.sql │ │ │ ├── q31.sql │ │ │ ├── q32.sql │ │ │ ├── q33.sql │ │ │ ├── q34.sql │ │ │ ├── q35.sql │ │ │ ├── q36.sql │ │ │ ├── q37.sql │ │ │ ├── q38.sql │ │ │ ├── q39.sql │ │ │ ├── q4.sql │ │ │ ├── q40.sql │ │ │ ├── q41.sql │ │ │ ├── q42.sql │ │ │ ├── q43.sql │ │ │ ├── q44.sql │ │ │ ├── q45.sql │ │ │ ├── q46.sql │ │ │ ├── q47.sql │ │ │ ├── q48.sql │ │ │ ├── q49.sql │ │ │ ├── q5.sql │ │ │ ├── q50.sql │ │ │ ├── q51.sql │ │ │ ├── q52.sql │ │ │ ├── q53.sql │ │ │ ├── q54.sql │ │ │ ├── q55.sql │ │ │ ├── q56.sql │ │ │ ├── q57.sql │ │ │ ├── q58.sql │ │ │ ├── q59.sql │ │ │ ├── q6.sql │ │ │ ├── q60.sql │ │ │ ├── q61.sql │ │ │ ├── q62.sql │ │ │ ├── q63.sql │ │ │ ├── q64.sql │ │ │ ├── q65.sql │ │ │ ├── q66.sql │ │ │ ├── q67.sql │ │ │ ├── q68.sql │ │ │ ├── q69.sql │ │ │ ├── q7.sql │ │ │ ├── q70.sql │ │ │ ├── q71.sql │ │ │ ├── q72.sql │ │ │ ├── q73.sql │ │ │ ├── q74.sql │ │ │ ├── q75.sql │ │ │ ├── q76.sql │ │ │ ├── q77.sql │ │ │ ├── q78.sql │ │ │ ├── q79.sql │ │ │ ├── q8.sql │ │ │ ├── q80.sql │ │ │ ├── q81.sql │ │ │ ├── q82.sql │ │ │ ├── q83.sql │ │ │ ├── q84.sql │ │ │ ├── q85.sql │ │ │ ├── q86.sql │ │ │ ├── q87.sql │ │ │ ├── q88.sql │ │ │ ├── q89.sql │ │ │ ├── q9.sql │ │ │ ├── q90.sql │ │ │ ├── q91.sql │ │ │ ├── q92.sql │ │ │ ├── q93.sql │ │ │ ├── q94.sql │ │ │ ├── q95.sql │ │ │ ├── q96.sql │ │ │ ├── q97.sql │ │ │ ├── q98.sql │ │ │ └── q99.sql │ │ └── tpch/ │ │ ├── q1.sql │ │ ├── q10.sql │ │ ├── q11.sql │ │ ├── q12.sql │ │ ├── q13.sql │ │ ├── q14.sql │ │ ├── q15.sql │ │ ├── q16.sql │ │ ├── q17.sql │ │ ├── q18.sql │ │ ├── q19.sql │ │ ├── q2.sql │ │ ├── q20.sql │ │ ├── q21.sql │ │ ├── q22.sql │ │ ├── q3.sql │ │ ├── q4.sql │ │ ├── q5.sql │ │ ├── q6.sql │ │ ├── q7.sql │ │ ├── q8.sql │ │ └── q9.sql │ ├── run.py │ └── tpcbench.py ├── common/ │ ├── pom.xml │ └── src/ │ ├── main/ │ │ ├── java/ │ │ │ └── org/ │ │ │ └── apache/ │ │ │ ├── arrow/ │ │ │ │ └── c/ │ │ │ │ ├── AbstractCometSchemaImporter.java │ │ │ │ └── ArrowImporter.java │ │ │ └── comet/ │ │ │ ├── CometNativeException.java │ │ │ ├── CometOutOfMemoryError.java │ │ │ ├── CometRuntimeException.java │ │ │ ├── CometSchemaImporter.java │ │ │ ├── IcebergApi.java │ │ │ ├── NativeBase.java │ │ │ ├── ParquetRuntimeException.java │ │ │ ├── exceptions/ │ │ │ │ └── CometQueryExecutionException.java │ │ │ ├── parquet/ │ │ │ │ ├── AbstractColumnReader.java │ │ │ │ ├── ArrowConstantColumnReader.java │ │ │ │ ├── ArrowRowIndexColumnReader.java │ │ │ │ ├── BloomFilterReader.java │ │ │ │ ├── ColumnIndexReader.java │ │ │ │ ├── ColumnPageReader.java │ │ │ │ ├── ColumnReader.java │ │ │ │ ├── CometFileKeyUnwrapper.java │ │ │ │ ├── CometInputFile.java │ │ │ │ ├── DictionaryPageReader.java │ │ │ │ ├── FileReader.java │ │ │ │ ├── FooterReader.java │ │ │ │ ├── IcebergCometNativeBatchReader.java │ │ │ │ ├── IndexFilter.java │ │ │ │ ├── LazyColumnReader.java │ │ │ │ ├── Native.java │ │ │ │ ├── NativeBatchReader.java │ │ │ │ ├── NativeColumnReader.java │ │ │ │ ├── ParquetColumnSpec.java │ │ │ │ ├── ParquetMetadataSerializer.java │ │ │ │ ├── ReadOptions.java │ │ │ │ ├── RowGroupFilter.java │ │ │ │ ├── RowGroupReader.java │ │ │ │ ├── TypeUtil.java │ │ │ │ ├── Utils.java │ │ │ │ ├── WrappedInputFile.java │ │ │ │ └── WrappedSeekableInputStream.java │ │ │ └── vector/ │ │ │ ├── CometDecodedVector.java │ │ │ ├── CometDelegateVector.java │ │ │ ├── CometDictionary.java │ │ │ ├── CometDictionaryVector.java │ │ │ ├── CometLazyVector.java │ │ │ ├── CometListVector.java │ │ │ ├── CometMapVector.java │ │ │ ├── CometPlainVector.java │ │ │ ├── CometSelectionVector.java │ │ │ ├── CometStructVector.java │ │ │ └── CometVector.java │ │ ├── resources/ │ │ │ └── log4j2.properties │ │ ├── scala/ │ │ │ └── org/ │ │ │ └── apache/ │ │ │ ├── comet/ │ │ │ │ ├── CometConf.scala │ │ │ │ ├── Constants.scala │ │ │ │ ├── objectstore/ │ │ │ │ │ └── NativeConfig.scala │ │ │ │ ├── package.scala │ │ │ │ ├── parquet/ │ │ │ │ │ ├── CometParquetUtils.scala │ │ │ │ │ └── CometReaderThreadPool.scala │ │ │ │ └── vector/ │ │ │ │ ├── NativeUtil.scala │ │ │ │ └── StreamReader.scala │ │ │ └── spark/ │ │ │ └── sql/ │ │ │ └── comet/ │ │ │ ├── CastOverflowException.scala │ │ │ ├── execution/ │ │ │ │ └── arrow/ │ │ │ │ ├── ArrowReaderIterator.scala │ │ │ │ ├── ArrowWriters.scala │ │ │ │ └── CometArrowConverters.scala │ │ │ ├── parquet/ │ │ │ │ ├── CometParquetReadSupport.scala │ │ │ │ └── CometSparkToParquetSchemaConverter.scala │ │ │ └── util/ │ │ │ └── Utils.scala │ │ ├── spark-3.4/ │ │ │ └── org/ │ │ │ └── apache/ │ │ │ ├── comet/ │ │ │ │ └── shims/ │ │ │ │ ├── ShimBatchReader.scala │ │ │ │ └── ShimFileFormat.scala │ │ │ └── spark/ │ │ │ └── sql/ │ │ │ └── comet/ │ │ │ └── shims/ │ │ │ └── ShimTaskMetrics.scala │ │ ├── spark-3.5/ │ │ │ └── org/ │ │ │ └── apache/ │ │ │ ├── comet/ │ │ │ │ └── shims/ │ │ │ │ ├── ShimBatchReader.scala │ │ │ │ └── ShimFileFormat.scala │ │ │ └── spark/ │ │ │ └── sql/ │ │ │ └── comet/ │ │ │ └── shims/ │ │ │ └── ShimTaskMetrics.scala │ │ ├── spark-3.x/ │ │ │ └── org/ │ │ │ └── apache/ │ │ │ └── comet/ │ │ │ └── shims/ │ │ │ ├── CometTypeShim.scala │ │ │ └── ShimCometConf.scala │ │ └── spark-4.0/ │ │ └── org/ │ │ └── apache/ │ │ ├── comet/ │ │ │ └── shims/ │ │ │ ├── CometTypeShim.scala │ │ │ ├── ShimBatchReader.scala │ │ │ ├── ShimCometConf.scala │ │ │ └── ShimFileFormat.scala │ │ └── spark/ │ │ └── sql/ │ │ └── comet/ │ │ └── shims/ │ │ └── ShimTaskMetrics.scala │ └── test/ │ ├── java/ │ │ └── org/ │ │ └── apache/ │ │ └── comet/ │ │ └── parquet/ │ │ ├── TestColumnReader.java │ │ ├── TestCometInputFile.java │ │ ├── TestFileReader.java │ │ └── TestUtils.java │ └── resources/ │ ├── log4j.properties │ └── log4j2.properties ├── conf/ │ └── log4rs.yaml ├── dev/ │ ├── cargo.config │ ├── changelog/ │ │ ├── 0.1.0.md │ │ ├── 0.10.0.md │ │ ├── 0.11.0.md │ │ ├── 0.12.0.md │ │ ├── 0.13.0.md │ │ ├── 0.14.0.md │ │ ├── 0.14.1.md │ │ ├── 0.2.0.md │ │ ├── 0.3.0.md │ │ ├── 0.4.0.md │ │ ├── 0.5.0.md │ │ ├── 0.6.0.md │ │ ├── 0.7.0.md │ │ ├── 0.8.0.md │ │ ├── 0.9.0.md │ │ └── 0.9.1.md │ ├── checkstyle-suppressions.xml │ ├── ci/ │ │ ├── check-suites.py │ │ └── check-working-tree-clean.sh │ ├── copyright/ │ │ └── java-header.txt │ ├── diffs/ │ │ ├── 3.4.3.diff │ │ ├── 3.5.8.diff │ │ ├── 4.0.1.diff │ │ └── iceberg/ │ │ ├── 1.10.0.diff │ │ ├── 1.8.1.diff │ │ └── 1.9.1.diff │ ├── ensure-jars-have-correct-contents.sh │ ├── generate-release-docs.sh │ ├── regenerate-golden-files.sh │ ├── release/ │ │ ├── build-release-comet.sh │ │ ├── check-rat-report.py │ │ ├── comet-rm/ │ │ │ ├── Dockerfile │ │ │ └── build-comet-native-libs.sh │ │ ├── create-tarball.sh │ │ ├── generate-changelog.py │ │ ├── publish-to-maven.sh │ │ ├── rat_exclude_files.txt │ │ ├── release-tarball.sh │ │ ├── requirements.txt │ │ ├── run-rat.sh │ │ ├── verify-release-candidate.sh │ │ └── verifying-release-candidates.md │ └── scalastyle-config.xml ├── docs/ │ ├── .gitignore │ ├── Makefile │ ├── README.md │ ├── build.sh │ ├── generate-versions.py │ ├── make.bat │ ├── requirements.txt │ ├── source/ │ │ ├── _static/ │ │ │ └── theme_overrides.css │ │ ├── _templates/ │ │ │ ├── docs-sidebar.html │ │ │ └── layout.html │ │ ├── about/ │ │ │ ├── gluten_comparison.md │ │ │ └── index.md │ │ ├── asf/ │ │ │ └── index.md │ │ ├── conf.py │ │ ├── contributor-guide/ │ │ │ ├── adding_a_new_expression.md │ │ │ ├── adding_a_new_operator.md │ │ │ ├── benchmark-results/ │ │ │ │ ├── blaze-0.5.0-tpcds.json │ │ │ │ ├── blaze-0.5.0-tpch.json │ │ │ │ ├── gluten-1.4.0-tpcds.json │ │ │ │ ├── gluten-1.4.0-tpch.json │ │ │ │ ├── spark-3.5.3-tpcds.json │ │ │ │ ├── spark-3.5.3-tpch.json │ │ │ │ ├── tpc-ds.md │ │ │ │ └── tpc-h.md │ │ │ ├── benchmarking.md │ │ │ ├── benchmarking_aws_ec2.md │ │ │ ├── benchmarking_macos.md │ │ │ ├── benchmarking_spark_sql_perf.md │ │ │ ├── bug_triage.md │ │ │ ├── contributing.md │ │ │ ├── debugging.md │ │ │ ├── development.md │ │ │ ├── expression-audit-log.md │ │ │ ├── ffi.md │ │ │ ├── iceberg-spark-tests.md │ │ │ ├── index.md │ │ │ ├── jvm_shuffle.md │ │ │ ├── native_shuffle.md │ │ │ ├── parquet_scans.md │ │ │ ├── plugin_overview.md │ │ │ ├── profiling.md │ │ │ ├── release_process.md │ │ │ ├── roadmap.md │ │ │ ├── spark-sql-tests.md │ │ │ ├── sql-file-tests.md │ │ │ ├── sql_error_propagation.md │ │ │ └── tracing.md │ │ ├── index.md │ │ └── user-guide/ │ │ ├── index.md │ │ └── latest/ │ │ ├── compatibility.md │ │ ├── configs.md │ │ ├── datasources.md │ │ ├── datatypes.md │ │ ├── expressions.md │ │ ├── iceberg.md │ │ ├── index.rst │ │ ├── installation.md │ │ ├── kubernetes.md │ │ ├── metrics.md │ │ ├── operators.md │ │ ├── source.md │ │ └── tuning.md │ └── spark_expressions_support.md ├── fuzz-testing/ │ ├── .gitignore │ ├── README.md │ ├── pom.xml │ ├── run.sh │ └── src/ │ └── main/ │ └── scala/ │ └── org/ │ └── apache/ │ └── comet/ │ └── fuzz/ │ ├── ComparisonTool.scala │ ├── Main.scala │ ├── Meta.scala │ ├── QueryGen.scala │ ├── QueryRunner.scala │ └── Utils.scala ├── kube/ │ ├── Dockerfile │ └── local/ │ ├── hadoop.env │ └── hdfs-docker-compose.yml ├── mvnw ├── mvnw.cmd ├── native/ │ ├── Cargo.toml │ ├── README.md │ ├── common/ │ │ ├── Cargo.toml │ │ ├── README.md │ │ └── src/ │ │ ├── bin/ │ │ │ └── analyze_trace.rs │ │ ├── error.rs │ │ ├── lib.rs │ │ ├── query_context.rs │ │ ├── tracing.rs │ │ └── utils.rs │ ├── core/ │ │ ├── Cargo.toml │ │ ├── benches/ │ │ │ ├── array_element_append.rs │ │ │ ├── bit_util.rs │ │ │ ├── common.rs │ │ │ ├── parquet_decode.rs │ │ │ ├── parquet_read.rs │ │ │ └── perf.rs │ │ └── src/ │ │ ├── common/ │ │ │ ├── bit.rs │ │ │ ├── buffer.rs │ │ │ └── mod.rs │ │ ├── execution/ │ │ │ ├── columnar_to_row.rs │ │ │ ├── expressions/ │ │ │ │ ├── arithmetic.rs │ │ │ │ ├── bitwise.rs │ │ │ │ ├── comparison.rs │ │ │ │ ├── logical.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── nullcheck.rs │ │ │ │ ├── partition.rs │ │ │ │ ├── random.rs │ │ │ │ ├── strings.rs │ │ │ │ ├── subquery.rs │ │ │ │ └── temporal.rs │ │ │ ├── jni_api.rs │ │ │ ├── memory_pools/ │ │ │ │ ├── config.rs │ │ │ │ ├── fair_pool.rs │ │ │ │ ├── logging_pool.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── task_shared.rs │ │ │ │ └── unified_pool.rs │ │ │ ├── metrics/ │ │ │ │ ├── mod.rs │ │ │ │ └── utils.rs │ │ │ ├── mod.rs │ │ │ ├── operators/ │ │ │ │ ├── copy.rs │ │ │ │ ├── csv_scan.rs │ │ │ │ ├── expand.rs │ │ │ │ ├── iceberg_scan.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── parquet_writer.rs │ │ │ │ ├── projection.rs │ │ │ │ ├── scan.rs │ │ │ │ └── shuffle_scan.rs │ │ │ ├── planner/ │ │ │ │ ├── expression_registry.rs │ │ │ │ ├── macros.rs │ │ │ │ └── operator_registry.rs │ │ │ ├── planner.rs │ │ │ ├── serde.rs │ │ │ ├── sort.rs │ │ │ ├── spark_config.rs │ │ │ ├── spark_plan.rs │ │ │ ├── tracing.rs │ │ │ └── utils.rs │ │ ├── lib.rs │ │ └── parquet/ │ │ ├── cast_column.rs │ │ ├── data_type.rs │ │ ├── encryption_support.rs │ │ ├── mod.rs │ │ ├── mutable_vector.rs │ │ ├── objectstore/ │ │ │ ├── mod.rs │ │ │ └── s3.rs │ │ ├── parquet_exec.rs │ │ ├── parquet_read_cached_factory.rs │ │ ├── parquet_support.rs │ │ ├── read/ │ │ │ ├── column.rs │ │ │ ├── levels.rs │ │ │ ├── mod.rs │ │ │ └── values.rs │ │ ├── schema_adapter.rs │ │ └── util/ │ │ ├── bit_packing.rs │ │ ├── buffer.rs │ │ ├── jni.rs │ │ ├── memory.rs │ │ ├── mod.rs │ │ └── test_common/ │ │ ├── mod.rs │ │ ├── page_util.rs │ │ └── rand_gen.rs │ ├── fs-hdfs/ │ │ ├── Cargo.toml │ │ ├── LICENSE.txt │ │ ├── README.md │ │ ├── build.rs │ │ ├── c_src/ │ │ │ ├── libhdfs/ │ │ │ │ ├── config.h │ │ │ │ ├── exception.c │ │ │ │ ├── exception.h │ │ │ │ ├── hdfs.c │ │ │ │ ├── hdfs.h │ │ │ │ ├── htable.c │ │ │ │ ├── htable.h │ │ │ │ ├── jni_helper.c │ │ │ │ ├── jni_helper.h │ │ │ │ └── os/ │ │ │ │ ├── mutexes.h │ │ │ │ ├── posix/ │ │ │ │ │ ├── mutexes.c │ │ │ │ │ ├── platform.h │ │ │ │ │ ├── thread.c │ │ │ │ │ └── thread_local_storage.c │ │ │ │ ├── thread.h │ │ │ │ └── thread_local_storage.h │ │ │ ├── libminidfs/ │ │ │ │ ├── native_mini_dfs.c │ │ │ │ └── native_mini_dfs.h │ │ │ └── wrapper.h │ │ ├── rustfmt.toml │ │ └── src/ │ │ ├── err.rs │ │ ├── hdfs.rs │ │ ├── lib.rs │ │ ├── minidfs.rs │ │ ├── native.rs │ │ ├── util.rs │ │ └── walkdir/ │ │ ├── mod.rs │ │ └── tree_iter.rs │ ├── hdfs/ │ │ ├── Cargo.toml │ │ ├── README.md │ │ └── src/ │ │ ├── lib.rs │ │ └── object_store/ │ │ ├── hdfs.rs │ │ └── mod.rs │ ├── jni-bridge/ │ │ ├── Cargo.toml │ │ ├── README.md │ │ ├── src/ │ │ │ ├── batch_iterator.rs │ │ │ ├── comet_exec.rs │ │ │ ├── comet_metric_node.rs │ │ │ ├── comet_task_memory_manager.rs │ │ │ ├── errors.rs │ │ │ ├── lib.rs │ │ │ └── shuffle_block_iterator.rs │ │ └── testdata/ │ │ ├── backtrace.txt │ │ └── stacktrace.txt │ ├── proto/ │ │ ├── Cargo.toml │ │ ├── README.md │ │ ├── build.rs │ │ └── src/ │ │ ├── lib.rs │ │ └── proto/ │ │ ├── config.proto │ │ ├── expr.proto │ │ ├── literal.proto │ │ ├── metric.proto │ │ ├── operator.proto │ │ ├── partitioning.proto │ │ └── types.proto │ ├── rustfmt.toml │ ├── shuffle/ │ │ ├── Cargo.toml │ │ ├── README.md │ │ ├── benches/ │ │ │ ├── row_columnar.rs │ │ │ └── shuffle_writer.rs │ │ └── src/ │ │ ├── bin/ │ │ │ └── shuffle_bench.rs │ │ ├── comet_partitioning.rs │ │ ├── ipc.rs │ │ ├── lib.rs │ │ ├── metrics.rs │ │ ├── partitioners/ │ │ │ ├── empty_schema.rs │ │ │ ├── mod.rs │ │ │ ├── multi_partition.rs │ │ │ ├── partitioned_batch_iterator.rs │ │ │ ├── single_partition.rs │ │ │ └── traits.rs │ │ ├── shuffle_writer.rs │ │ ├── spark_crc32c_hasher.rs │ │ ├── spark_unsafe/ │ │ │ ├── list.rs │ │ │ ├── map.rs │ │ │ ├── mod.rs │ │ │ ├── row.rs │ │ │ └── unsafe_object.rs │ │ └── writers/ │ │ ├── buf_batch_writer.rs │ │ ├── checksum.rs │ │ ├── mod.rs │ │ ├── shuffle_block_writer.rs │ │ └── spill.rs │ └── spark-expr/ │ ├── Cargo.toml │ ├── README.md │ ├── benches/ │ │ ├── aggregate.rs │ │ ├── bloom_filter_agg.rs │ │ ├── cast_from_boolean.rs │ │ ├── cast_from_string.rs │ │ ├── cast_int_to_timestamp.rs │ │ ├── cast_non_int_numeric_timestamp.rs │ │ ├── cast_numeric.rs │ │ ├── conditional.rs │ │ ├── date_trunc.rs │ │ ├── decimal_div.rs │ │ ├── normalize_nan.rs │ │ ├── padding.rs │ │ ├── to_csv.rs │ │ └── wide_decimal.rs │ ├── src/ │ │ ├── agg_funcs/ │ │ │ ├── avg.rs │ │ │ ├── avg_decimal.rs │ │ │ ├── correlation.rs │ │ │ ├── covariance.rs │ │ │ ├── mod.rs │ │ │ ├── stddev.rs │ │ │ ├── sum_decimal.rs │ │ │ ├── sum_int.rs │ │ │ └── variance.rs │ │ ├── array_funcs/ │ │ │ ├── array_compact.rs │ │ │ ├── array_insert.rs │ │ │ ├── get_array_struct_fields.rs │ │ │ ├── list_extract.rs │ │ │ ├── mod.rs │ │ │ └── size.rs │ │ ├── bloom_filter/ │ │ │ ├── bit.rs │ │ │ ├── bloom_filter_agg.rs │ │ │ ├── bloom_filter_might_contain.rs │ │ │ ├── mod.rs │ │ │ ├── spark_bit_array.rs │ │ │ └── spark_bloom_filter.rs │ │ ├── comet_scalar_funcs.rs │ │ ├── conditional_funcs/ │ │ │ ├── if_expr.rs │ │ │ └── mod.rs │ │ ├── conversion_funcs/ │ │ │ ├── boolean.rs │ │ │ ├── cast.rs │ │ │ ├── mod.rs │ │ │ ├── numeric.rs │ │ │ ├── string.rs │ │ │ ├── temporal.rs │ │ │ └── utils.rs │ │ ├── csv_funcs/ │ │ │ ├── csv_write_options.rs │ │ │ ├── mod.rs │ │ │ └── to_csv.rs │ │ ├── datetime_funcs/ │ │ │ ├── date_diff.rs │ │ │ ├── date_from_unix_date.rs │ │ │ ├── date_trunc.rs │ │ │ ├── extract_date_part.rs │ │ │ ├── hours.rs │ │ │ ├── make_date.rs │ │ │ ├── mod.rs │ │ │ ├── timestamp_trunc.rs │ │ │ └── unix_timestamp.rs │ │ ├── error.rs │ │ ├── hash_funcs/ │ │ │ ├── mod.rs │ │ │ ├── murmur3.rs │ │ │ ├── utils.rs │ │ │ └── xxhash64.rs │ │ ├── json_funcs/ │ │ │ ├── from_json.rs │ │ │ ├── mod.rs │ │ │ └── to_json.rs │ │ ├── kernels/ │ │ │ ├── mod.rs │ │ │ ├── strings.rs │ │ │ └── temporal.rs │ │ ├── lib.rs │ │ ├── math_funcs/ │ │ │ ├── abs.rs │ │ │ ├── ceil.rs │ │ │ ├── checked_arithmetic.rs │ │ │ ├── div.rs │ │ │ ├── floor.rs │ │ │ ├── internal/ │ │ │ │ ├── checkoverflow.rs │ │ │ │ ├── decimal_rescale_check.rs │ │ │ │ ├── make_decimal.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── normalize_nan.rs │ │ │ │ └── unscaled_value.rs │ │ │ ├── log.rs │ │ │ ├── mod.rs │ │ │ ├── modulo_expr.rs │ │ │ ├── negative.rs │ │ │ ├── round.rs │ │ │ ├── unhex.rs │ │ │ ├── utils.rs │ │ │ └── wide_decimal_binary_expr.rs │ │ ├── nondetermenistic_funcs/ │ │ │ ├── internal/ │ │ │ │ ├── mod.rs │ │ │ │ └── rand_utils.rs │ │ │ ├── mod.rs │ │ │ ├── monotonically_increasing_id.rs │ │ │ ├── rand.rs │ │ │ └── randn.rs │ │ ├── predicate_funcs/ │ │ │ ├── is_nan.rs │ │ │ ├── mod.rs │ │ │ └── rlike.rs │ │ ├── query_context.rs │ │ ├── static_invoke/ │ │ │ ├── char_varchar_utils/ │ │ │ │ ├── mod.rs │ │ │ │ └── read_side_padding.rs │ │ │ └── mod.rs │ │ ├── string_funcs/ │ │ │ ├── contains.rs │ │ │ ├── get_json_object.rs │ │ │ ├── mod.rs │ │ │ ├── split.rs │ │ │ └── substring.rs │ │ ├── struct_funcs/ │ │ │ ├── create_named_struct.rs │ │ │ ├── get_struct_field.rs │ │ │ └── mod.rs │ │ ├── test_common/ │ │ │ ├── file_util.rs │ │ │ └── mod.rs │ │ ├── timezone.rs │ │ ├── unbound.rs │ │ └── utils.rs │ └── tests/ │ └── spark_expr_reg.rs ├── pom.xml ├── rust-toolchain.toml ├── scalafmt.conf ├── spark/ │ ├── README.md │ ├── inspections/ │ │ ├── CometTPCDSQueriesList-results.txt │ │ └── CometTPCHQueriesList-results.txt │ ├── pom.xml │ └── src/ │ ├── main/ │ │ ├── java/ │ │ │ └── org/ │ │ │ └── apache/ │ │ │ ├── comet/ │ │ │ │ ├── CometBatchIterator.java │ │ │ │ ├── CometShuffleBlockIterator.java │ │ │ │ └── NativeColumnarToRowInfo.java │ │ │ ├── parquet/ │ │ │ │ └── filter2/ │ │ │ │ └── predicate/ │ │ │ │ └── SparkFilterApi.java │ │ │ └── spark/ │ │ │ ├── CometTaskMemoryManager.java │ │ │ ├── shuffle/ │ │ │ │ ├── comet/ │ │ │ │ │ ├── CometBoundedShuffleMemoryAllocator.java │ │ │ │ │ ├── CometShuffleChecksumSupport.java │ │ │ │ │ ├── CometShuffleMemoryAllocator.java │ │ │ │ │ ├── CometShuffleMemoryAllocatorTrait.java │ │ │ │ │ ├── CometUnifiedShuffleMemoryAllocator.java │ │ │ │ │ └── TooLargePageException.java │ │ │ │ └── sort/ │ │ │ │ ├── CometShuffleExternalSorter.java │ │ │ │ ├── CometShuffleExternalSorterAsync.java │ │ │ │ ├── CometShuffleExternalSorterSync.java │ │ │ │ └── SpillSorter.java │ │ │ └── sql/ │ │ │ └── comet/ │ │ │ ├── CometScalarSubquery.java │ │ │ └── execution/ │ │ │ └── shuffle/ │ │ │ ├── CometBypassMergeSortShuffleWriter.java │ │ │ ├── CometDiskBlockWriter.java │ │ │ ├── CometUnsafeShuffleWriter.java │ │ │ ├── ExposedByteArrayOutputStream.java │ │ │ ├── ShuffleThreadPool.java │ │ │ ├── SpillInfo.java │ │ │ └── SpillWriter.java │ │ ├── resources/ │ │ │ └── log4j2.properties │ │ ├── scala/ │ │ │ └── org/ │ │ │ └── apache/ │ │ │ ├── comet/ │ │ │ │ ├── CometExecIterator.scala │ │ │ │ ├── CometFallback.scala │ │ │ │ ├── CometMetricsListener.scala │ │ │ │ ├── CometSparkSessionExtensions.scala │ │ │ │ ├── DataTypeSupport.scala │ │ │ │ ├── ExtendedExplainInfo.scala │ │ │ │ ├── GenerateDocs.scala │ │ │ │ ├── MetricsSupport.scala │ │ │ │ ├── Native.scala │ │ │ │ ├── NativeColumnarToRowConverter.scala │ │ │ │ ├── SparkErrorConverter.scala │ │ │ │ ├── Tracing.scala │ │ │ │ ├── expressions/ │ │ │ │ │ ├── CometCast.scala │ │ │ │ │ ├── CometEvalMode.scala │ │ │ │ │ └── RegExp.scala │ │ │ │ ├── iceberg/ │ │ │ │ │ └── IcebergReflection.scala │ │ │ │ ├── parquet/ │ │ │ │ │ ├── CometParquetFileFormat.scala │ │ │ │ │ ├── ParquetFilters.scala │ │ │ │ │ └── SourceFilterSerde.scala │ │ │ │ ├── rules/ │ │ │ │ │ ├── CometExecRule.scala │ │ │ │ │ ├── CometScanRule.scala │ │ │ │ │ ├── EliminateRedundantTransitions.scala │ │ │ │ │ └── RewriteJoin.scala │ │ │ │ ├── serde/ │ │ │ │ │ ├── CometAggregateExpressionSerde.scala │ │ │ │ │ ├── CometBloomFilterMightContain.scala │ │ │ │ │ ├── CometExpressionSerde.scala │ │ │ │ │ ├── CometOperatorSerde.scala │ │ │ │ │ ├── CometScalarFunction.scala │ │ │ │ │ ├── CometScalarSubquery.scala │ │ │ │ │ ├── CometSortOrder.scala │ │ │ │ │ ├── QueryPlanSerde.scala │ │ │ │ │ ├── SupportLevel.scala │ │ │ │ │ ├── aggregates.scala │ │ │ │ │ ├── arithmetic.scala │ │ │ │ │ ├── arrays.scala │ │ │ │ │ ├── bitwise.scala │ │ │ │ │ ├── collectionOperations.scala │ │ │ │ │ ├── conditional.scala │ │ │ │ │ ├── contraintExpressions.scala │ │ │ │ │ ├── datetime.scala │ │ │ │ │ ├── decimalExpressions.scala │ │ │ │ │ ├── hash.scala │ │ │ │ │ ├── literals.scala │ │ │ │ │ ├── maps.scala │ │ │ │ │ ├── math.scala │ │ │ │ │ ├── namedExpressions.scala │ │ │ │ │ ├── nondetermenistic.scala │ │ │ │ │ ├── operator/ │ │ │ │ │ │ ├── CometDataWritingCommand.scala │ │ │ │ │ │ ├── CometIcebergNativeScan.scala │ │ │ │ │ │ ├── CometNativeScan.scala │ │ │ │ │ │ ├── CometSink.scala │ │ │ │ │ │ └── package.scala │ │ │ │ │ ├── predicates.scala │ │ │ │ │ ├── statics.scala │ │ │ │ │ ├── strings.scala │ │ │ │ │ ├── structs.scala │ │ │ │ │ └── unixtime.scala │ │ │ │ └── testing/ │ │ │ │ ├── FuzzDataGenerator.scala │ │ │ │ └── ParquetGenerator.scala │ │ │ └── spark/ │ │ │ ├── CometSource.scala │ │ │ ├── Plugins.scala │ │ │ ├── shuffle/ │ │ │ │ └── sort/ │ │ │ │ └── RowPartition.scala │ │ │ └── sql/ │ │ │ └── comet/ │ │ │ ├── CometBatchScanExec.scala │ │ │ ├── CometBroadcastExchangeExec.scala │ │ │ ├── CometCoalesceExec.scala │ │ │ ├── CometCollectLimitExec.scala │ │ │ ├── CometColumnarToRowExec.scala │ │ │ ├── CometCsvNativeScanExec.scala │ │ │ ├── CometExecRDD.scala │ │ │ ├── CometExecUtils.scala │ │ │ ├── CometIcebergNativeScanExec.scala │ │ │ ├── CometLocalTableScanExec.scala │ │ │ ├── CometMetricNode.scala │ │ │ ├── CometNativeColumnarToRowExec.scala │ │ │ ├── CometNativeScanExec.scala │ │ │ ├── CometNativeWriteExec.scala │ │ │ ├── CometPlan.scala │ │ │ ├── CometScanExec.scala │ │ │ ├── CometScanUtils.scala │ │ │ ├── CometSparkToColumnarExec.scala │ │ │ ├── CometTakeOrderedAndProjectExec.scala │ │ │ ├── CometWindowExec.scala │ │ │ ├── DecimalPrecision.scala │ │ │ ├── execution/ │ │ │ │ └── shuffle/ │ │ │ │ ├── CometBlockStoreShuffleReader.scala │ │ │ │ ├── CometNativeShuffleWriter.scala │ │ │ │ ├── CometShuffleDependency.scala │ │ │ │ ├── CometShuffleExchangeExec.scala │ │ │ │ ├── CometShuffleManager.scala │ │ │ │ ├── CometShuffledRowRDD.scala │ │ │ │ └── NativeBatchDecoderIterator.scala │ │ │ ├── operators.scala │ │ │ └── plans/ │ │ │ └── AliasAwareOutputExpression.scala │ │ ├── spark-3.4/ │ │ │ └── org/ │ │ │ └── apache/ │ │ │ ├── comet/ │ │ │ │ └── shims/ │ │ │ │ ├── CometExprShim.scala │ │ │ │ ├── ShimCometBroadcastExchangeExec.scala │ │ │ │ ├── ShimSQLConf.scala │ │ │ │ └── ShimSubqueryBroadcast.scala │ │ │ └── spark/ │ │ │ └── sql/ │ │ │ └── comet/ │ │ │ └── shims/ │ │ │ ├── ShimCometScanExec.scala │ │ │ └── ShimSparkErrorConverter.scala │ │ ├── spark-3.5/ │ │ │ └── org/ │ │ │ └── apache/ │ │ │ ├── comet/ │ │ │ │ └── shims/ │ │ │ │ ├── CometExprShim.scala │ │ │ │ ├── ShimCometBroadcastExchangeExec.scala │ │ │ │ ├── ShimSQLConf.scala │ │ │ │ └── ShimSubqueryBroadcast.scala │ │ │ └── spark/ │ │ │ └── sql/ │ │ │ └── comet/ │ │ │ └── shims/ │ │ │ ├── ShimCometScanExec.scala │ │ │ └── ShimSparkErrorConverter.scala │ │ ├── spark-3.x/ │ │ │ └── org/ │ │ │ └── apache/ │ │ │ ├── comet/ │ │ │ │ └── shims/ │ │ │ │ ├── ShimCometShuffleExchangeExec.scala │ │ │ │ └── ShimCometSparkSessionExtensions.scala │ │ │ └── spark/ │ │ │ ├── comet/ │ │ │ │ └── shims/ │ │ │ │ └── ShimCometDriverPlugin.scala │ │ │ └── sql/ │ │ │ ├── ExtendedExplainGenerator.scala │ │ │ └── comet/ │ │ │ └── shims/ │ │ │ ├── ShimCometShuffleWriteProcessor.scala │ │ │ └── ShimStreamSourceAwareSparkPlan.scala │ │ └── spark-4.0/ │ │ └── org/ │ │ └── apache/ │ │ ├── comet/ │ │ │ └── shims/ │ │ │ ├── CometExprShim.scala │ │ │ ├── ShimCometBroadcastExchangeExec.scala │ │ │ ├── ShimCometShuffleExchangeExec.scala │ │ │ ├── ShimCometSparkSessionExtensions.scala │ │ │ ├── ShimSQLConf.scala │ │ │ └── ShimSubqueryBroadcast.scala │ │ └── spark/ │ │ ├── comet/ │ │ │ └── shims/ │ │ │ └── ShimCometDriverPlugin.scala │ │ └── sql/ │ │ └── comet/ │ │ └── shims/ │ │ ├── ShimCometScanExec.scala │ │ ├── ShimCometShuffleWriteProcessor.scala │ │ ├── ShimSparkErrorConverter.scala │ │ └── ShimStreamSourceAwareSparkPlan.scala │ └── test/ │ ├── java/ │ │ └── org/ │ │ └── apache/ │ │ ├── comet/ │ │ │ ├── IntegrationTestSuite.java │ │ │ └── hadoop/ │ │ │ └── fs/ │ │ │ └── FakeHDFSFileSystem.java │ │ └── iceberg/ │ │ └── rest/ │ │ └── RESTCatalogAdapter.java │ ├── resources/ │ │ ├── log4j.properties │ │ ├── log4j2.properties │ │ ├── sql-tests/ │ │ │ └── expressions/ │ │ │ ├── aggregate/ │ │ │ │ ├── aggregate_filter.sql │ │ │ │ ├── avg.sql │ │ │ │ ├── bit_agg.sql │ │ │ │ ├── corr.sql │ │ │ │ ├── count.sql │ │ │ │ ├── covariance.sql │ │ │ │ ├── first_last.sql │ │ │ │ ├── min_max.sql │ │ │ │ ├── stddev.sql │ │ │ │ ├── sum.sql │ │ │ │ └── variance.sql │ │ │ ├── array/ │ │ │ │ ├── array_append.sql │ │ │ │ ├── array_compact.sql │ │ │ │ ├── array_concat.sql │ │ │ │ ├── array_contains.sql │ │ │ │ ├── array_distinct.sql │ │ │ │ ├── array_except.sql │ │ │ │ ├── array_filter.sql │ │ │ │ ├── array_insert.sql │ │ │ │ ├── array_insert_legacy.sql │ │ │ │ ├── array_intersect.sql │ │ │ │ ├── array_join.sql │ │ │ │ ├── array_max.sql │ │ │ │ ├── array_min.sql │ │ │ │ ├── array_remove.sql │ │ │ │ ├── array_repeat.sql │ │ │ │ ├── array_union.sql │ │ │ │ ├── arrays_overlap.sql │ │ │ │ ├── create_array.sql │ │ │ │ ├── element_at.sql │ │ │ │ ├── element_at_ansi.sql │ │ │ │ ├── flatten.sql │ │ │ │ ├── get_array_item.sql │ │ │ │ ├── get_array_item_ansi.sql │ │ │ │ ├── get_array_struct_fields.sql │ │ │ │ ├── size.sql │ │ │ │ └── sort_array.sql │ │ │ ├── bitwise/ │ │ │ │ └── bitwise.sql │ │ │ ├── cast/ │ │ │ │ ├── cast.sql │ │ │ │ ├── cast_decimal_to_primitive.sql │ │ │ │ └── cast_double_to_string.sql │ │ │ ├── conditional/ │ │ │ │ ├── boolean.sql │ │ │ │ ├── case_when.sql │ │ │ │ ├── coalesce.sql │ │ │ │ ├── if_expr.sql │ │ │ │ ├── in_set.sql │ │ │ │ ├── is_not_null.sql │ │ │ │ ├── is_null.sql │ │ │ │ └── predicates.sql │ │ │ ├── datetime/ │ │ │ │ ├── date_add.sql │ │ │ │ ├── date_diff.sql │ │ │ │ ├── date_format.sql │ │ │ │ ├── date_format_enabled.sql │ │ │ │ ├── date_from_unix_date.sql │ │ │ │ ├── date_sub.sql │ │ │ │ ├── datetime.sql │ │ │ │ ├── from_unix_time.sql │ │ │ │ ├── from_unix_time_enabled.sql │ │ │ │ ├── hour.sql │ │ │ │ ├── last_day.sql │ │ │ │ ├── make_date.sql │ │ │ │ ├── minute.sql │ │ │ │ ├── next_day.sql │ │ │ │ ├── second.sql │ │ │ │ ├── trunc_date.sql │ │ │ │ ├── trunc_timestamp.sql │ │ │ │ ├── unix_date.sql │ │ │ │ └── unix_timestamp.sql │ │ │ ├── decimal/ │ │ │ │ ├── decimal_div.sql │ │ │ │ ├── decimal_div_ansi.sql │ │ │ │ └── decimal_ops.sql │ │ │ ├── hash/ │ │ │ │ ├── crc32.sql │ │ │ │ └── hash.sql │ │ │ ├── map/ │ │ │ │ ├── get_map_value.sql │ │ │ │ ├── map_contains_key.sql │ │ │ │ ├── map_entries.sql │ │ │ │ ├── map_from_arrays.sql │ │ │ │ ├── map_from_entries.sql │ │ │ │ ├── map_keys.sql │ │ │ │ └── map_values.sql │ │ │ ├── math/ │ │ │ │ ├── abs.sql │ │ │ │ ├── abs_ansi.sql │ │ │ │ ├── acos.sql │ │ │ │ ├── arithmetic.sql │ │ │ │ ├── arithmetic_ansi.sql │ │ │ │ ├── asin.sql │ │ │ │ ├── atan.sql │ │ │ │ ├── atan2.sql │ │ │ │ ├── bin.sql │ │ │ │ ├── ceil.sql │ │ │ │ ├── cos.sql │ │ │ │ ├── cosh.sql │ │ │ │ ├── cot.sql │ │ │ │ ├── exp.sql │ │ │ │ ├── expm1.sql │ │ │ │ ├── floor.sql │ │ │ │ ├── isnan.sql │ │ │ │ ├── log.sql │ │ │ │ ├── log10.sql │ │ │ │ ├── log2.sql │ │ │ │ ├── pow.sql │ │ │ │ ├── round.sql │ │ │ │ ├── signum.sql │ │ │ │ ├── sin.sql │ │ │ │ ├── sinh.sql │ │ │ │ ├── sqrt.sql │ │ │ │ ├── tan.sql │ │ │ │ └── tanh.sql │ │ │ ├── misc/ │ │ │ │ ├── parquet_default_values.sql │ │ │ │ ├── scalar_subquery.sql │ │ │ │ └── width_bucket.sql │ │ │ ├── string/ │ │ │ │ ├── ascii.sql │ │ │ │ ├── bit_length.sql │ │ │ │ ├── chr.sql │ │ │ │ ├── concat.sql │ │ │ │ ├── concat_ws.sql │ │ │ │ ├── contains.sql │ │ │ │ ├── ends_with.sql │ │ │ │ ├── get_json_object.sql │ │ │ │ ├── hex.sql │ │ │ │ ├── init_cap.sql │ │ │ │ ├── init_cap_enabled.sql │ │ │ │ ├── left.sql │ │ │ │ ├── length.sql │ │ │ │ ├── like.sql │ │ │ │ ├── lower.sql │ │ │ │ ├── lower_enabled.sql │ │ │ │ ├── luhn_check.sql │ │ │ │ ├── octet_length.sql │ │ │ │ ├── regexp_replace.sql │ │ │ │ ├── regexp_replace_enabled.sql │ │ │ │ ├── reverse.sql │ │ │ │ ├── right.sql │ │ │ │ ├── rlike.sql │ │ │ │ ├── rlike_enabled.sql │ │ │ │ ├── starts_with.sql │ │ │ │ ├── string.sql │ │ │ │ ├── string_instr.sql │ │ │ │ ├── string_lpad.sql │ │ │ │ ├── string_repeat.sql │ │ │ │ ├── string_replace.sql │ │ │ │ ├── string_rpad.sql │ │ │ │ ├── string_space.sql │ │ │ │ ├── string_translate.sql │ │ │ │ ├── string_trim.sql │ │ │ │ ├── substring.sql │ │ │ │ ├── unhex.sql │ │ │ │ ├── upper.sql │ │ │ │ └── upper_enabled.sql │ │ │ ├── struct/ │ │ │ │ ├── create_named_struct.sql │ │ │ │ ├── get_struct_field.sql │ │ │ │ ├── json_to_structs.sql │ │ │ │ └── structs_to_json.sql │ │ │ └── window/ │ │ │ └── lag_lead.sql │ │ ├── test-data/ │ │ │ ├── before_1582_date_v2_4_5.snappy.parquet │ │ │ ├── before_1582_date_v2_4_6.snappy.parquet │ │ │ ├── before_1582_date_v3_2_0.snappy.parquet │ │ │ ├── before_1582_timestamp_int96_dict_v2_4_5.snappy.parquet │ │ │ ├── before_1582_timestamp_int96_dict_v2_4_6.snappy.parquet │ │ │ ├── before_1582_timestamp_int96_dict_v3_2_0.snappy.parquet │ │ │ ├── before_1582_timestamp_int96_plain_v2_4_5.snappy.parquet │ │ │ ├── before_1582_timestamp_int96_plain_v2_4_6.snappy.parquet │ │ │ ├── before_1582_timestamp_int96_plain_v3_2_0.snappy.parquet │ │ │ ├── before_1582_timestamp_micros_v2_4_5.snappy.parquet │ │ │ ├── before_1582_timestamp_micros_v2_4_6.snappy.parquet │ │ │ ├── before_1582_timestamp_micros_v3_2_0.snappy.parquet │ │ │ ├── before_1582_timestamp_millis_v2_4_5.snappy.parquet │ │ │ ├── before_1582_timestamp_millis_v2_4_6.snappy.parquet │ │ │ ├── before_1582_timestamp_millis_v3_2_0.snappy.parquet │ │ │ ├── csv-test-1.csv │ │ │ ├── csv-test-2.csv │ │ │ ├── dec-in-fixed-len.parquet │ │ │ ├── decimal32-written-as-64-bit-dict.snappy.parquet │ │ │ ├── decimal32-written-as-64-bit.snappy.parquet │ │ │ └── json-test-1.ndjson │ │ ├── tpcds-extended/ │ │ │ └── q72.sql │ │ ├── tpcds-micro-benchmarks/ │ │ │ ├── add_decimals.sql │ │ │ ├── add_many_decimals.sql │ │ │ ├── add_many_integers.sql │ │ │ ├── agg_high_cardinality.sql │ │ │ ├── agg_low_cardinality.sql │ │ │ ├── agg_stddev.sql │ │ │ ├── agg_sum_decimals_no_grouping.sql │ │ │ ├── agg_sum_integers_no_grouping.sql │ │ │ ├── agg_sum_integers_with_grouping.sql │ │ │ ├── case_when_column_or_null.sql │ │ │ ├── case_when_scalar.sql │ │ │ ├── char_type.sql │ │ │ ├── explode.sql │ │ │ ├── filter_highly_selective.sql │ │ │ ├── filter_less_selective.sql │ │ │ ├── if_column_or_null.sql │ │ │ ├── join_anti.sql │ │ │ ├── join_condition.sql │ │ │ ├── join_exploding_output.sql │ │ │ ├── join_inner.sql │ │ │ ├── join_left_outer.sql │ │ │ ├── join_semi.sql │ │ │ ├── rlike.sql │ │ │ ├── scan_decimal.sql │ │ │ └── to_json.sql │ │ ├── tpcds-plan-stability/ │ │ │ ├── approved-plans-v1_4/ │ │ │ │ ├── q1.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q1.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q10.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q10.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q11.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q11.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q12.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q12.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q13.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q13.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q14a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q14a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q14b.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q14b.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q15.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q15.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q16.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q16.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q17.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q17.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q18.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q18.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q19.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q19.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q2.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q2.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q20.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q20.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q21.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q21.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q22.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q22.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q23a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q23a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q23b.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q23b.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q24a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q24a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q24b.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q24b.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q25.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q25.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q26.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q26.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q27.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q27.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q28.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q28.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q29.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q29.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q3.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q3.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q30.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q30.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q31.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q31.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q32.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q32.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q33.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q33.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q34.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q34.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q35.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q35.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q36.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q36.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q37.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q37.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q38.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q38.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q39a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q39a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q39b.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q39b.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q4.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q4.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q40.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q40.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q41.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q41.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q42.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q42.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q43.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q43.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q44.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q44.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q45.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q45.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q46.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q46.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q47.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q47.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q48.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q48.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q49.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q49.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q5.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q5.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q50.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q50.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q51.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q51.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q52.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q52.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q53.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q53.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q54.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q54.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q55.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q55.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q56.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q56.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q57.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q57.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q58.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q58.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q59.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q59.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q6.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q6.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q60.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q60.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q61.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q61.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q62.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q62.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q63.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q63.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q64.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q64.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q65.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q65.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q66.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q66.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q67.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q67.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q68.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q68.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q69.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q69.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q7.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q7.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q70.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q70.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q71.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q71.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q72.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q72.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q73.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q73.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q74.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q74.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q75.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q75.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q76.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q76.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q77.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q77.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q78.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q78.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q79.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q79.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q8.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q8.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q80.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q80.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q81.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q81.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q82.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q82.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q83.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q83.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q84.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q84.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q85.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q85.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q86.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q86.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q87.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q87.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q88.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q88.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q89.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q89.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q9.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q9.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q90.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q90.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q91.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q91.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q92.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q92.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q93.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q93.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q94.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q94.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q95.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q95.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q96.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q96.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q97.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q97.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q98.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q98.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q99.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ └── q99.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── approved-plans-v1_4-spark3_5/ │ │ │ │ ├── q1.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q1.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q10.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q10.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q11.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q11.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q12.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q12.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q13.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q13.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q14a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q14a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q14b.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q14b.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q15.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q15.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q16.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q16.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q17.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q17.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q18.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q18.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q19.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q19.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q2.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q2.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q20.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q20.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q21.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q21.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q22.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q22.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q23a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q23a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q23b.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q23b.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q24a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q24a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q24b.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q24b.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q25.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q25.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q26.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q26.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q27.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q27.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q28.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q28.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q29.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q29.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q3.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q3.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q30.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q30.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q31.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q31.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q32.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q32.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q33.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q33.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q34.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q34.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q35.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q35.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q36.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q36.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q37.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q37.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q38.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q38.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q39a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q39a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q39b.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q39b.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q4.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q4.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q40.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q40.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q41.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q41.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q42.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q42.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q43.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q43.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q44.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q44.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q45.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q45.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q46.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q46.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q47.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q47.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q48.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q48.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q49.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q49.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q5.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q5.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q50.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q50.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q51.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q51.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q52.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q52.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q53.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q53.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q54.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q54.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q55.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q55.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q56.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q56.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q57.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q57.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q58.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q58.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q59.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q59.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q6.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q6.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q60.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q60.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q61.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q61.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q62.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q62.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q63.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q63.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q64.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q64.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q65.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q65.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q66.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q66.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q67.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q67.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q68.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q68.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q69.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q69.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q7.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q7.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q70.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q70.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q71.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q71.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q72.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q72.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q73.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q73.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q74.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q74.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q75.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q75.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q76.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q76.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q77.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q77.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q78.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q78.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q79.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q79.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q8.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q8.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q80.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q80.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q81.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q81.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q82.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q82.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q83.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q83.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q84.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q84.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q85.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q85.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q86.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q86.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q87.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q87.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q88.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q88.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q89.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q89.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q9.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q9.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q90.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q90.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q91.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q91.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q92.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q92.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q93.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q93.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q94.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q94.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q95.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q95.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q96.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q96.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q97.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q97.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q98.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q98.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q99.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ └── q99.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── approved-plans-v1_4-spark4_0/ │ │ │ │ ├── q1.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q1.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q10.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q10.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q11.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q11.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q12.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q12.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q13.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q13.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q14a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q14a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q14b.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q14b.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q15.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q15.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q16.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q16.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q17.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q17.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q18.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q18.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q19.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q19.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q2.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q2.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q20.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q20.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q21.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q21.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q22.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q22.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q23a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q23a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q23b.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q23b.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q24a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q24a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q24b.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q24b.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q25.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q25.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q26.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q26.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q27.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q27.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q28.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q28.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q29.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q29.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q3.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q3.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q30.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q30.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q31.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q31.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q32.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q32.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q33.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q33.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q34.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q34.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q35.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q35.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q36.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q36.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q37.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q37.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q38.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q38.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q39a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q39a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q39b.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q39b.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q4.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q4.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q40.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q40.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q41.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q41.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q42.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q42.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q43.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q43.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q44.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q44.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q45.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q45.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q46.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q46.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q47.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q47.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q48.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q48.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q49.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q49.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q5.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q5.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q50.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q50.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q51.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q51.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q52.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q52.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q53.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q53.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q54.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q54.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q55.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q55.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q56.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q56.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q57.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q57.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q58.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q58.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q59.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q59.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q6.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q6.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q60.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q60.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q61.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q61.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q62.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q62.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q63.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q63.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q64.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q64.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q65.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q65.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q66.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q66.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q67.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q67.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q68.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q68.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q69.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q69.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q7.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q7.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q70.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q70.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q71.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q71.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q72.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q72.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q73.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q73.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q74.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q74.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q75.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q75.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q76.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q76.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q77.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q77.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q78.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q78.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q79.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q79.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q8.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q8.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q80.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q80.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q81.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q81.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q82.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q82.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q83.ansi.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q83.ansi.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q84.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q84.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q85.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q85.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q86.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q86.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q87.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q87.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q88.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q88.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q89.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q89.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q9.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q9.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q90.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q90.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q91.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q91.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q92.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q92.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q93.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q93.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q94.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q94.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q95.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q95.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q96.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q96.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q97.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q97.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q98.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q98.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q99.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ └── q99.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── approved-plans-v2_7/ │ │ │ │ ├── q10a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q10a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q11.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q11.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q12.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q12.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q14.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q14.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q14a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q14a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q18a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q18a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q20.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q20.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q22.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q22.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q22a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q22a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q24.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q24.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q27a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q27a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q34.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q34.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q35.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q35.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q35a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q35a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q36a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q36a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q47.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q47.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q49.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q49.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q51a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q51a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q57.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q57.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q5a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q5a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q6.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q6.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q64.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q64.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q67a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q67a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q70a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q70a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q72.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q72.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q74.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q74.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q75.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q75.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q77a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q77a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q78.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q78.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q80a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q80a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q86a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q86a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q98.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ └── q98.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── approved-plans-v2_7-spark3_5/ │ │ │ │ ├── q10a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q10a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q11.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q11.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q12.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q12.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q14.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q14.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q14a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q14a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q18a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q18a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q20.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q20.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q22.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q22.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q22a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q22a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q24.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q24.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q27a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q27a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q34.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q34.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q35.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q35.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q35a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q35a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q36a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q36a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q47.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q47.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q49.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q49.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q51a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q51a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q57.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q57.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q5a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q5a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q6.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q6.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q64.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q64.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q67a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q67a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q70a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q70a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q72.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q72.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q74.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q74.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q75.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q75.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q77a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q77a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q78.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q78.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q80a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q80a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q86a.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q86a.native_iceberg_compat/ │ │ │ │ │ └── extended.txt │ │ │ │ ├── q98.native_datafusion/ │ │ │ │ │ └── extended.txt │ │ │ │ └── q98.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ └── approved-plans-v2_7-spark4_0/ │ │ │ ├── q10a.native_datafusion/ │ │ │ │ └── extended.txt │ │ │ ├── q10a.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── q11.native_datafusion/ │ │ │ │ └── extended.txt │ │ │ ├── q11.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── q12.native_datafusion/ │ │ │ │ └── extended.txt │ │ │ ├── q12.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── q14.native_datafusion/ │ │ │ │ └── extended.txt │ │ │ ├── q14.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── q14a.native_datafusion/ │ │ │ │ └── extended.txt │ │ │ ├── q14a.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── q18a.native_datafusion/ │ │ │ │ └── extended.txt │ │ │ ├── q18a.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── q20.native_datafusion/ │ │ │ │ └── extended.txt │ │ │ ├── q20.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── q22.native_datafusion/ │ │ │ │ └── extended.txt │ │ │ ├── q22.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── q22a.native_datafusion/ │ │ │ │ └── extended.txt │ │ │ ├── q22a.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── q24.native_datafusion/ │ │ │ │ └── extended.txt │ │ │ ├── q24.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── q27a.native_datafusion/ │ │ │ │ └── extended.txt │ │ │ ├── q27a.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── q34.native_datafusion/ │ │ │ │ └── extended.txt │ │ │ ├── q34.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── q35.native_datafusion/ │ │ │ │ └── extended.txt │ │ │ ├── q35.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── q35a.native_datafusion/ │ │ │ │ └── extended.txt │ │ │ ├── q35a.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── q36a.native_datafusion/ │ │ │ │ └── extended.txt │ │ │ ├── q36a.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── q47.native_datafusion/ │ │ │ │ └── extended.txt │ │ │ ├── q47.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── q49.native_datafusion/ │ │ │ │ └── extended.txt │ │ │ ├── q49.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── q51a.native_datafusion/ │ │ │ │ └── extended.txt │ │ │ ├── q51a.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── q57.native_datafusion/ │ │ │ │ └── extended.txt │ │ │ ├── q57.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── q5a.native_datafusion/ │ │ │ │ └── extended.txt │ │ │ ├── q5a.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── q6.native_datafusion/ │ │ │ │ └── extended.txt │ │ │ ├── q6.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── q64.native_datafusion/ │ │ │ │ └── extended.txt │ │ │ ├── q64.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── q67a.native_datafusion/ │ │ │ │ └── extended.txt │ │ │ ├── q67a.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── q70a.native_datafusion/ │ │ │ │ └── extended.txt │ │ │ ├── q70a.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── q72.native_datafusion/ │ │ │ │ └── extended.txt │ │ │ ├── q72.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── q74.native_datafusion/ │ │ │ │ └── extended.txt │ │ │ ├── q74.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── q75.native_datafusion/ │ │ │ │ └── extended.txt │ │ │ ├── q75.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── q77a.native_datafusion/ │ │ │ │ └── extended.txt │ │ │ ├── q77a.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── q78.native_datafusion/ │ │ │ │ └── extended.txt │ │ │ ├── q78.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── q80a.native_datafusion/ │ │ │ │ └── extended.txt │ │ │ ├── q80a.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── q86a.native_datafusion/ │ │ │ │ └── extended.txt │ │ │ ├── q86a.native_iceberg_compat/ │ │ │ │ └── extended.txt │ │ │ ├── q98.native_datafusion/ │ │ │ │ └── extended.txt │ │ │ └── q98.native_iceberg_compat/ │ │ │ └── extended.txt │ │ ├── tpcds-query-results/ │ │ │ ├── extended/ │ │ │ │ └── q72.sql.out │ │ │ ├── v1_4/ │ │ │ │ ├── q1.sql.out │ │ │ │ ├── q10.sql.out │ │ │ │ ├── q11.sql.out │ │ │ │ ├── q12.sql.out │ │ │ │ ├── q13.sql.out │ │ │ │ ├── q14a.sql.out │ │ │ │ ├── q14b.sql.out │ │ │ │ ├── q15.sql.out │ │ │ │ ├── q16.sql.out │ │ │ │ ├── q17.sql.out │ │ │ │ ├── q18.sql.out │ │ │ │ ├── q19.sql.out │ │ │ │ ├── q2.sql.out │ │ │ │ ├── q20.sql.out │ │ │ │ ├── q21.sql.out │ │ │ │ ├── q22.sql.out │ │ │ │ ├── q23a.sql.out │ │ │ │ ├── q23b.sql.out │ │ │ │ ├── q24a.sql.out │ │ │ │ ├── q24b.sql.out │ │ │ │ ├── q25.sql.out │ │ │ │ ├── q26.sql.out │ │ │ │ ├── q27.sql.out │ │ │ │ ├── q28.sql.out │ │ │ │ ├── q29.sql.out │ │ │ │ ├── q3.sql.out │ │ │ │ ├── q30.sql.out │ │ │ │ ├── q31.sql.out │ │ │ │ ├── q32.sql.out │ │ │ │ ├── q33.sql.out │ │ │ │ ├── q34.sql.out │ │ │ │ ├── q35.sql.out │ │ │ │ ├── q36.sql.out │ │ │ │ ├── q37.sql.out │ │ │ │ ├── q38.sql.out │ │ │ │ ├── q39a.sql.out │ │ │ │ ├── q39b.sql.out │ │ │ │ ├── q4.sql.out │ │ │ │ ├── q40.sql.out │ │ │ │ ├── q41.sql.out │ │ │ │ ├── q42.sql.out │ │ │ │ ├── q43.sql.out │ │ │ │ ├── q44.sql.out │ │ │ │ ├── q45.sql.out │ │ │ │ ├── q46.sql.out │ │ │ │ ├── q47.sql.out │ │ │ │ ├── q48.sql.out │ │ │ │ ├── q49.sql.out │ │ │ │ ├── q5.sql.out │ │ │ │ ├── q50.sql.out │ │ │ │ ├── q51.sql.out │ │ │ │ ├── q52.sql.out │ │ │ │ ├── q53.sql.out │ │ │ │ ├── q54.sql.out │ │ │ │ ├── q55.sql.out │ │ │ │ ├── q56.sql.out │ │ │ │ ├── q57.sql.out │ │ │ │ ├── q58.sql.out │ │ │ │ ├── q59.sql.out │ │ │ │ ├── q6.sql.out │ │ │ │ ├── q60.sql.out │ │ │ │ ├── q61.sql.out │ │ │ │ ├── q62.sql.out │ │ │ │ ├── q63.sql.out │ │ │ │ ├── q64.sql.out │ │ │ │ ├── q65.sql.out │ │ │ │ ├── q66.sql.out │ │ │ │ ├── q67.sql.out │ │ │ │ ├── q68.sql.out │ │ │ │ ├── q69.sql.out │ │ │ │ ├── q7.sql.out │ │ │ │ ├── q70.sql.out │ │ │ │ ├── q71.sql.out │ │ │ │ ├── q72.sql.out │ │ │ │ ├── q73.sql.out │ │ │ │ ├── q74.sql.out │ │ │ │ ├── q75.sql.out │ │ │ │ ├── q76.sql.out │ │ │ │ ├── q77.sql.out │ │ │ │ ├── q78.sql.out │ │ │ │ ├── q79.sql.out │ │ │ │ ├── q8.sql.out │ │ │ │ ├── q80.sql.out │ │ │ │ ├── q81.sql.out │ │ │ │ ├── q82.sql.out │ │ │ │ ├── q83.sql.out │ │ │ │ ├── q84.sql.out │ │ │ │ ├── q85.sql.out │ │ │ │ ├── q86.sql.out │ │ │ │ ├── q87.sql.out │ │ │ │ ├── q88.sql.out │ │ │ │ ├── q89.sql.out │ │ │ │ ├── q9.sql.out │ │ │ │ ├── q90.sql.out │ │ │ │ ├── q91.sql.out │ │ │ │ ├── q92.sql.out │ │ │ │ ├── q93.sql.out │ │ │ │ ├── q94.sql.out │ │ │ │ ├── q95.sql.out │ │ │ │ ├── q96.sql.out │ │ │ │ ├── q97.sql.out │ │ │ │ ├── q98.sql.out │ │ │ │ └── q99.sql.out │ │ │ ├── v2_7/ │ │ │ │ ├── q10a.sql.out │ │ │ │ ├── q11.sql.out │ │ │ │ ├── q12.sql.out │ │ │ │ ├── q14.sql.out │ │ │ │ ├── q14a.sql.out │ │ │ │ ├── q18a.sql.out │ │ │ │ ├── q20.sql.out │ │ │ │ ├── q22.sql.out │ │ │ │ ├── q22a.sql.out │ │ │ │ ├── q24.sql.out │ │ │ │ ├── q27a.sql.out │ │ │ │ ├── q34.sql.out │ │ │ │ ├── q35.sql.out │ │ │ │ ├── q35a.sql.out │ │ │ │ ├── q36a.sql.out │ │ │ │ ├── q47.sql.out │ │ │ │ ├── q49.sql.out │ │ │ │ ├── q51a.sql.out │ │ │ │ ├── q57.sql.out │ │ │ │ ├── q5a.sql.out │ │ │ │ ├── q6.sql.out │ │ │ │ ├── q64.sql.out │ │ │ │ ├── q67a.sql.out │ │ │ │ ├── q70a.sql.out │ │ │ │ ├── q72.sql.out │ │ │ │ ├── q74.sql.out │ │ │ │ ├── q75.sql.out │ │ │ │ ├── q77a.sql.out │ │ │ │ ├── q78.sql.out │ │ │ │ ├── q80a.sql.out │ │ │ │ ├── q86a.sql.out │ │ │ │ └── q98.sql.out │ │ │ └── v2_7-spark4_0/ │ │ │ └── q36a.sql.out │ │ ├── tpch-extended/ │ │ │ └── q1.sql │ │ └── tpch-query-results/ │ │ ├── q1.sql.out │ │ ├── q10.sql.out │ │ ├── q11.sql.out │ │ ├── q12.sql.out │ │ ├── q13.sql.out │ │ ├── q14.sql.out │ │ ├── q15.sql.out │ │ ├── q16.sql.out │ │ ├── q17.sql.out │ │ ├── q18.sql.out │ │ ├── q19.sql.out │ │ ├── q2.sql.out │ │ ├── q20.sql.out │ │ ├── q21.sql.out │ │ ├── q22.sql.out │ │ ├── q3.sql.out │ │ ├── q4.sql.out │ │ ├── q5.sql.out │ │ ├── q6.sql.out │ │ ├── q7.sql.out │ │ ├── q8.sql.out │ │ └── q9.sql.out │ ├── scala/ │ │ └── org/ │ │ └── apache/ │ │ ├── comet/ │ │ │ ├── CometArrayExpressionSuite.scala │ │ │ ├── CometBitwiseExpressionSuite.scala │ │ │ ├── CometCastSuite.scala │ │ │ ├── CometCsvExpressionSuite.scala │ │ │ ├── CometDateTimeUtilsSuite.scala │ │ │ ├── CometExpressionSuite.scala │ │ │ ├── CometFuzzAggregateSuite.scala │ │ │ ├── CometFuzzIcebergBase.scala │ │ │ ├── CometFuzzIcebergSuite.scala │ │ │ ├── CometFuzzMathSuite.scala │ │ │ ├── CometFuzzTestBase.scala │ │ │ ├── CometFuzzTestSuite.scala │ │ │ ├── CometHashExpressionSuite.scala │ │ │ ├── CometIcebergNativeSuite.scala │ │ │ ├── CometJsonExpressionSuite.scala │ │ │ ├── CometMapExpressionSuite.scala │ │ │ ├── CometMathExpressionSuite.scala │ │ │ ├── CometNativeSuite.scala │ │ │ ├── CometS3TestBase.scala │ │ │ ├── CometSparkSessionExtensionsSuite.scala │ │ │ ├── CometSqlFileTestSuite.scala │ │ │ ├── CometStringExpressionSuite.scala │ │ │ ├── CometTemporalExpressionSuite.scala │ │ │ ├── DataGenerator.scala │ │ │ ├── DataGeneratorSuite.scala │ │ │ ├── IcebergReadFromS3Suite.scala │ │ │ ├── SparkErrorConverterSuite.scala │ │ │ ├── SqlFileTestParser.scala │ │ │ ├── WithHdfsCluster.scala │ │ │ ├── csv/ │ │ │ │ └── CometCsvNativeReadSuite.scala │ │ │ ├── exec/ │ │ │ │ ├── CometAggregateSuite.scala │ │ │ │ ├── CometColumnarShuffleSuite.scala │ │ │ │ ├── CometExec3_4PlusSuite.scala │ │ │ │ ├── CometExecSuite.scala │ │ │ │ ├── CometGenerateExecSuite.scala │ │ │ │ ├── CometJoinSuite.scala │ │ │ │ ├── CometNativeColumnarToRowSuite.scala │ │ │ │ ├── CometNativeReaderSuite.scala │ │ │ │ ├── CometNativeShuffleSuite.scala │ │ │ │ └── CometWindowExecSuite.scala │ │ │ ├── expressions/ │ │ │ │ └── conditional/ │ │ │ │ ├── CometCaseWhenSuite.scala │ │ │ │ ├── CometCoalesceSuite.scala │ │ │ │ └── CometIfSuite.scala │ │ │ ├── objectstore/ │ │ │ │ └── NativeConfigSuite.scala │ │ │ ├── parquet/ │ │ │ │ ├── CometParquetWriterSuite.scala │ │ │ │ ├── ParquetReadFromFakeHadoopFsSuite.scala │ │ │ │ ├── ParquetReadFromS3Suite.scala │ │ │ │ └── ParquetReadSuite.scala │ │ │ └── rules/ │ │ │ ├── CometExecRuleSuite.scala │ │ │ └── CometScanRuleSuite.scala │ │ └── spark/ │ │ ├── CometPluginsSuite.scala │ │ ├── shuffle/ │ │ │ └── sort/ │ │ │ └── SpillSorterSuite.scala │ │ └── sql/ │ │ ├── CometSQLQueryTestHelper.scala │ │ ├── CometTPCDSQueriesList.scala │ │ ├── CometTPCDSQuerySuite.scala │ │ ├── CometTPCDSQueryTestSuite.scala │ │ ├── CometTPCHQueriesList.scala │ │ ├── CometTPCHQuerySuite.scala │ │ ├── CometTPCQueryBase.scala │ │ ├── CometTPCQueryListBase.scala │ │ ├── CometTestBase.scala │ │ ├── GenTPCHData.scala │ │ ├── TPCDSQueries.scala │ │ ├── TPCH.scala │ │ ├── Tables.scala │ │ ├── benchmark/ │ │ │ ├── CometAggregateExpressionBenchmark.scala │ │ │ ├── CometArithmeticBenchmark.scala │ │ │ ├── CometArrayExpressionBenchmark.scala │ │ │ ├── CometBenchmarkBase.scala │ │ │ ├── CometCastBooleanBenchmark.scala │ │ │ ├── CometCastNumericToNumericBenchmark.scala │ │ │ ├── CometCastNumericToStringBenchmark.scala │ │ │ ├── CometCastNumericToTemporalBenchmark.scala │ │ │ ├── CometCastStringToNumericBenchmark.scala │ │ │ ├── CometCastStringToTemporalBenchmark.scala │ │ │ ├── CometCastTemporalToNumericBenchmark.scala │ │ │ ├── CometCastTemporalToStringBenchmark.scala │ │ │ ├── CometCastTemporalToTemporalBenchmark.scala │ │ │ ├── CometColumnarToRowBenchmark.scala │ │ │ ├── CometComparisonExpressionBenchmark.scala │ │ │ ├── CometConditionalExpressionBenchmark.scala │ │ │ ├── CometCsvExpressionBenchmark.scala │ │ │ ├── CometDatetimeExpressionBenchmark.scala │ │ │ ├── CometExecBenchmark.scala │ │ │ ├── CometGetJsonObjectBenchmark.scala │ │ │ ├── CometHashExpressionBenchmark.scala │ │ │ ├── CometIcebergReadBenchmark.scala │ │ │ ├── CometJsonExpressionBenchmark.scala │ │ │ ├── CometOperatorSerdeBenchmark.scala │ │ │ ├── CometPartitionColumnBenchmark.scala │ │ │ ├── CometPredicateExpressionBenchmark.scala │ │ │ ├── CometReadBenchmark.scala │ │ │ ├── CometShuffleBenchmark.scala │ │ │ ├── CometStringExpressionBenchmark.scala │ │ │ ├── CometTPCDSMicroBenchmark.scala │ │ │ ├── CometTPCDSQueryBenchmark.scala │ │ │ ├── CometTPCHQueryBenchmark.scala │ │ │ └── CometTPCQueryBenchmarkBase.scala │ │ └── comet/ │ │ ├── CometDppFallbackRepro3949Suite.scala │ │ ├── CometPlanChecker.scala │ │ ├── CometPlanStabilitySuite.scala │ │ ├── CometShuffleFallbackStickinessSuite.scala │ │ ├── CometTaskMetricsSuite.scala │ │ └── ParquetEncryptionITCase.scala │ ├── spark-3.4/ │ │ └── org/ │ │ └── apache/ │ │ ├── comet/ │ │ │ └── shims/ │ │ │ └── ShimCometTPCHQuerySuite.scala │ │ └── spark/ │ │ └── sql/ │ │ └── ShimCometTestBase.scala │ ├── spark-3.5/ │ │ └── org/ │ │ └── apache/ │ │ ├── comet/ │ │ │ └── shims/ │ │ │ └── ShimCometTPCHQuerySuite.scala │ │ └── spark/ │ │ └── sql/ │ │ ├── CometToPrettyStringSuite.scala │ │ └── ShimCometTestBase.scala │ ├── spark-3.x/ │ │ └── org/ │ │ └── apache/ │ │ ├── comet/ │ │ │ └── iceberg/ │ │ │ └── RESTCatalogHelper.scala │ │ ├── iceberg/ │ │ │ └── rest/ │ │ │ └── RESTCatalogServlet.java │ │ └── spark/ │ │ └── sql/ │ │ └── comet/ │ │ └── shims/ │ │ └── ShimCometTPCDSQuerySuite.scala │ └── spark-4.0/ │ └── org/ │ └── apache/ │ ├── comet/ │ │ ├── exec/ │ │ │ └── CometShuffle4_0Suite.scala │ │ ├── iceberg/ │ │ │ └── RESTCatalogHelper.scala │ │ └── shims/ │ │ └── ShimCometTPCHQuerySuite.scala │ ├── iceberg/ │ │ └── rest/ │ │ └── RESTCatalogServlet.java │ └── spark/ │ ├── comet/ │ │ └── shims/ │ │ └── ShimTestUtils.scala │ └── sql/ │ ├── CometToPrettyStringSuite.scala │ ├── ShimCometTestBase.scala │ └── comet/ │ └── shims/ │ └── ShimCometTPCDSQuerySuite.scala └── spark-integration/ └── pom.xml ================================================ FILE CONTENTS ================================================ ================================================ FILE: .asf.yaml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # This file controls the settings of this repository # # See more details at # https://cwiki.apache.org/confluence/display/INFRA/Git+-+.asf.yaml+features notifications: commits: commits@datafusion.apache.org issues: github@datafusion.apache.org pullrequests: github@datafusion.apache.org discussions: github@datafusion.apache.org jira_options: link label worklog github: description: "Apache DataFusion Comet Spark Accelerator" homepage: https://datafusion.apache.org/comet labels: - arrow - datafusion - rust - spark enabled_merge_buttons: squash: true merge: false rebase: false features: issues: true discussions: true protected_branches: main: required_pull_request_reviews: required_approving_review_count: 1 pull_requests: allow_update_branch: true # publishes the content of the `asf-site` branch to # https://datafusion.apache.org/comet/ publish: whoami: asf-site subdir: comet ================================================ FILE: .claude/skills/audit-comet-expression/SKILL.md ================================================ --- name: audit-comet-expression description: Audit an existing Comet expression for correctness and test coverage. Studies the Spark implementation across versions 3.4.3, 3.5.8, and 4.0.1, reviews the Comet and DataFusion implementations, identifies missing test coverage, and offers to implement additional tests. argument-hint: --- Audit the Comet implementation of the `$ARGUMENTS` expression for correctness and test coverage. ## Overview This audit covers: 1. Spark implementation across versions 3.4.3, 3.5.8, and 4.0.1 2. Comet Scala serde implementation 3. Comet Rust / DataFusion implementation 4. Existing test coverage (SQL file tests and Scala tests) 5. Gap analysis and test recommendations --- ## Step 1: Locate the Spark Implementations Clone specific Spark version tags (use shallow clones to avoid polluting the workspace). Only clone a version if it is not already present. ```bash set -eu -o pipefail for tag in v3.4.3 v3.5.8 v4.0.1; do dir="/tmp/spark-${tag}" if [ ! -d "$dir" ]; then git clone --depth 1 --branch "$tag" https://github.com/apache/spark.git "$dir" fi done ``` ### Find the expression class in each Spark version Search the Catalyst SQL expressions source: ```bash for tag in v3.4.3 v3.5.8 v4.0.1; do dir="/tmp/spark-${tag}" echo "=== $tag ===" find "$dir/sql/catalyst/src/main/scala" -name "*.scala" | \ xargs grep -l "case class $ARGUMENTS\b\|object $ARGUMENTS\b" 2>/dev/null done ``` If the expression is not found in catalyst, also check core: ```bash for tag in v3.4.3 v3.5.8 v4.0.1; do dir="/tmp/spark-${tag}" echo "=== $tag ===" find "$dir/sql" -name "*.scala" | \ xargs grep -l "case class $ARGUMENTS\b\|object $ARGUMENTS\b" 2>/dev/null done ``` ### Read the Spark source for each version For each Spark version, read the expression file and note: - The `eval`, `nullSafeEval`, and `doGenCode` / `doGenCodeSafe` methods - The `inputTypes` and `dataType` fields (accepted input types, return type) - Null handling strategy (`nullable`, `nullSafeEval`) - ANSI mode behavior (`ansiEnabled`, `failOnError`) - Special cases, guards, `require` assertions, and runtime exceptions - Any constants or configuration the expression reads ### Compare across Spark versions Produce a concise diff summary of what changed between: - 3.4.3 → 3.5.8 - 3.5.8 → 4.0.1 Pay attention to: - New input types added or removed - Behavior changes for edge cases (null, overflow, empty, boundary) - New ANSI mode branches - New parameters or configuration - Breaking API changes that Comet must shim --- ## Step 2: Locate the Spark Tests ```bash for tag in v3.4.3 v3.5.8 v4.0.1; do dir="/tmp/spark-${tag}" echo "=== $tag ===" find "$dir/sql" -name "*.scala" -path "*/test/*" | \ xargs grep -l "$ARGUMENTS" 2>/dev/null done ``` Read the relevant Spark test files and produce a list of: - Input types covered - Edge cases exercised (null, empty, overflow, negative, boundary values, special characters, etc.) - ANSI mode tests - Error cases This list will be the reference for the coverage gap analysis in Step 5. --- ## Step 3: Locate the Comet Implementation ### Scala serde ```bash # Find the serde object grep -r "$ARGUMENTS" spark/src/main/scala/org/apache/comet/serde/ --include="*.scala" -l grep -r "$ARGUMENTS" spark/src/main/scala/org/apache/comet/ --include="*.scala" -l ``` Read the serde implementation and check: - Which Spark versions the serde handles - Whether `getSupportLevel` is implemented and accurate - Whether all input types are handled - Whether any types are explicitly marked `Unsupported` ### Shims ```bash find spark/src/main -name "CometExprShim.scala" | xargs grep -l "$ARGUMENTS" 2>/dev/null ``` If shims exist, read them and note any version-specific handling. ### Rust / DataFusion implementation ```bash # Search for the function in native/spark-expr grep -r "$ARGUMENTS" native/spark-expr/src/ --include="*.rs" -l grep -r "$ARGUMENTS" native/core/src/ --include="*.rs" -l ``` If the expression delegates to DataFusion, find it there too. Set `$DATAFUSION_SRC` to a local DataFusion checkout, or fall back to searching the cargo registry: ```bash if [ -n "${DATAFUSION_SRC:-}" ]; then grep -r "$ARGUMENTS" "$DATAFUSION_SRC" --include="*.rs" -l 2>/dev/null | head -10 else # Fall back to cargo registry (may include unrelated crates) grep -r "$ARGUMENTS" ~/.cargo/registry/src/*/datafusion* --include="*.rs" -l 2>/dev/null | head -10 fi ``` Read the Rust implementation and check: - Null handling (does it propagate nulls correctly?) - Overflow and underflow handling (returns `Err` vs panics) - Type dispatch (does it handle all types that Spark supports?) - ANSI / fail-on-error mode --- ## Step 4: Locate Existing Comet Tests ### SQL file tests ```bash # Find SQL test files for this expression find spark/src/test/resources/sql-tests/expressions/ -name "*.sql" | \ xargs grep -l "$ARGUMENTS" 2>/dev/null # Also check if there's a dedicated file find spark/src/test/resources/sql-tests/expressions/ -name "*$(echo $ARGUMENTS | tr '[:upper:]' '[:lower:]')*" ``` Read every SQL test file found and list: - Table schemas and data values used - Queries exercised - Query modes used (`query`, `spark_answer_only`, `tolerance`, `ignore`, `expect_error`) - Any ConfigMatrix directives ### Scala tests ```bash grep -r "$ARGUMENTS" spark/src/test/scala/ --include="*.scala" -l ``` Read the relevant Scala test files and list: - Input types covered - Edge cases exercised - Whether constant folding is disabled for literal tests --- ## Step 5: Gap Analysis Compare the Spark test coverage (Step 2) against the Comet test coverage (Step 4). Produce a structured gap report: ### Coverage matrix For each of the following dimensions, note whether it is covered in Comet tests or missing: | Dimension | Spark tests it | Comet SQL test | Comet Scala test | Gap? | | ------------------------------------------------------------------------------------------------------ | -------------- | -------------- | ---------------- | ---- | | Column reference argument(s) | | | | | | Literal argument(s) | | | | | | NULL input | | | | | | Empty string / empty array / empty map | | | | | | Array/map with NULL elements | | | | | | Zero, negative zero, negative values (numeric) | | | | | | Underflow, overflow | | | | | | Boundary values (INT_MIN, INT_MAX, Long.MinValue, minimum positive, etc.) | | | | | | NaN, Infinity, -Infinity, subnormal (float/double) | | | | | | Multibyte / special UTF-8 (composed vs decomposed, e.g. `é` U+00E9 vs `e` + U+0301, non-Latin scripts) | | | | | | ANSI mode (failOnError=true) | | | | | | Non-ANSI mode (failOnError=false) | | | | | | All supported input types | | | | | | Parquet dictionary encoding (ConfigMatrix) | | | | | | Cross-version behavior differences | | | | | ### Implementation gaps Also review the Comet implementation (Step 3) against the Spark behavior (Step 1): - Are there input types that Spark supports but `getSupportLevel` returns `Unsupported` without comment? - Are there behavioral differences that are NOT marked `Incompatible` but should be? - Are there behavioral differences between Spark versions that the Comet implementation does not account for (missing shim)? - Does the Rust implementation match the Spark behavior for all edge cases? --- ## Step 6: Recommendations Summarize findings as a prioritized list. ### High priority Issues where Comet may silently produce wrong results compared to Spark. ### Medium priority Missing test coverage for edge cases that could expose bugs. ### Low priority Minor gaps, cosmetic improvements, or nice-to-have tests. --- ## Step 7: Offer to Implement Missing Tests After presenting the gap analysis, ask the user: > I found the following missing test cases. Would you like me to implement them? > > - [list each missing test case] > > I can add them as SQL file tests in `spark/src/test/resources/sql-tests/expressions//$ARGUMENTS.sql` > (or as Scala tests in `CometExpressionSuite` for cases that require programmatic setup). If the user says yes, implement the missing tests following the SQL file test format described in `docs/source/contributor-guide/sql-file-tests.md`. Prefer SQL file tests over Scala tests. ### SQL file test template ```sql -- Licensed to the Apache Software Foundation (ASF) under one -- or more contributor license agreements. See the NOTICE file -- distributed with this work for additional information -- regarding copyright ownership. The ASF licenses this file -- to you under the Apache License, Version 2.0 (the -- "License"); you may not use this file except in compliance -- with the License. You may obtain a copy of the License at -- -- http://www.apache.org/licenses/LICENSE-2.0 -- -- Unless required by applicable law or agreed to in writing, -- software distributed under the License is distributed on an -- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -- KIND, either express or implied. See the License for the -- specific language governing permissions and limitations -- under the License. -- ConfigMatrix: parquet.enable.dictionary=false,true statement CREATE TABLE test_$ARGUMENTS(...) USING parquet statement INSERT INTO test_$ARGUMENTS VALUES (...), (NULL) -- column argument query SELECT $ARGUMENTS(col) FROM test_$ARGUMENTS -- literal arguments query SELECT $ARGUMENTS('value'), $ARGUMENTS(''), $ARGUMENTS(NULL) ``` ### Verify the tests pass After implementing tests, tell the user how to run them: ```bash ./mvnw test -DwildcardSuites="CometSqlFileTestSuite" -Dsuites="org.apache.comet.CometSqlFileTestSuite $ARGUMENTS" -Dtest=none ``` --- ## Step 8: Update the Expression Audit Log After completing the audit (whether or not tests were added), append a row to the audit log at `docs/source/contributor-guide/expression-audit-log.md`. The row should include: - Expression name - Spark versions checked (e.g. 3.4.3, 3.5.8, 4.0.1) - Today's date - A brief summary of findings (behavioral differences, bugs found/fixed, tests added, known incompatibilities) --- ## Output Format Present the audit as: 1. **Expression Summary** - Brief description of what `$ARGUMENTS` does, its input/output types, and null behavior 2. **Spark Version Differences** - Summary of any behavioral or API differences across Spark 3.4.3, 3.5.8, and 4.0.1 3. **Comet Implementation Notes** - Summary of how Comet implements this expression and any concerns 4. **Coverage Gap Analysis** - The gap table from Step 5, plus implementation gaps 5. **Recommendations** - Prioritized list from Step 6 6. **Offer to add tests** - The prompt from Step 7 ## Tone and Style - Write in clear, concise prose - Use backticks around code references (function names, file paths, class names, types, config keys) - Avoid robotic or formulaic language - Be constructive and acknowledge what is already well-covered before raising gaps - Avoid em dashes and semicolons; use separate sentences instead ================================================ FILE: .claude/skills/review-comet-pr/SKILL.md ================================================ --- name: review-comet-pr description: Review a DataFusion Comet pull request for Spark compatibility and implementation correctness. Provides guidance to a reviewer rather than posting comments directly. argument-hint: --- Review Comet PR #$ARGUMENTS ## Before You Start ### Gather PR Metadata Fetch the PR details to understand the scope: ```bash gh pr view $ARGUMENTS --repo apache/datafusion-comet --json title,body,author,isDraft,state,files ``` ### Review Existing Comments First Before forming your review: 1. **Read all existing review comments** on the PR 2. **Check the conversation tab** for any discussion 3. **Avoid duplicating feedback** that others have already provided 4. **Build on existing discussions** rather than starting new threads on the same topic 5. **If you have no additional concerns beyond what's already discussed, say so** 6. **Ignore Copilot reviews** - do not reference or build upon comments from GitHub Copilot ```bash # View existing comments on a PR gh pr view $ARGUMENTS --repo apache/datafusion-comet --comments ``` --- ## Review Workflow ### 1. Gather Context Read the changed files and understand the area of the codebase being modified: ```bash # View the diff gh pr diff $ARGUMENTS --repo apache/datafusion-comet ``` For expression PRs, check how similar expressions are implemented in the codebase. Look at the serde files in `spark/src/main/scala/org/apache/comet/serde/` and Rust implementations in `native/spark-expr/src/`. ### 2. Read Spark Source (Expression PRs) **For any PR that adds or modifies an expression, you must read the Spark source code to understand the canonical behavior.** This is the authoritative reference for what Comet must match. 1. **Clone or update the Spark repo:** ```bash # Clone if not already present (use /tmp to avoid polluting the workspace) if [ ! -d /tmp/spark ]; then git clone --depth 1 https://github.com/apache/spark.git /tmp/spark fi ``` 2. **Find the expression implementation in Spark:** ```bash # Search for the expression class (e.g., for "Conv", "Hex", "Substring") find /tmp/spark/sql/catalyst/src/main/scala -name "*.scala" | xargs grep -l "case class " ``` 3. **Read the Spark implementation carefully.** Pay attention to: - The `eval` and `doGenEval`/`nullSafeEval` methods. These define the exact behavior. - The `inputTypes` and `dataType` fields. These define which types Spark accepts and what it returns. - Null handling. Does it use `nullable = true`? Does `nullSafeEval` handle nulls implicitly? - Special cases, guards, and `require` assertions. - ANSI mode branches (look for `SQLConf.get.ansiEnabled` or `failOnError`). 4. **Read the Spark tests for the expression:** ```bash # Find test files find /tmp/spark/sql -name "*.scala" -path "*/test/*" | xargs grep -l "" ``` 5. **Compare the Spark behavior against the Comet implementation in the PR.** Identify: - Edge cases tested in Spark but not in the PR - Data types supported in Spark but not handled in the PR - Behavioral differences that should be marked `Incompatible` 6. **Suggest additional tests** for any edge cases or type combinations covered in Spark's tests that are missing from the PR's tests. ### 3. Spark Compatibility Check **This is the most critical aspect of Comet reviews.** Comet must produce identical results to Spark. For expression PRs, verify against the Spark source you read in step 2: 1. **Check edge cases** - Null handling - Overflow behavior - Empty input behavior - Type-specific behavior 2. **Verify all data types are handled** - Does Spark support this type? (Check `inputTypes` in Spark source) - Does the PR handle all Spark-supported types? 3. **Check for ANSI mode differences** - Spark behavior may differ between legacy and ANSI modes - PR should handle both or mark as `Incompatible` ### 4. Check Against Implementation Guidelines **Always verify PRs follow the implementation guidelines.** #### Scala Serde (`spark/src/main/scala/org/apache/comet/serde/`) - [ ] Expression class correctly identified - [ ] All child expressions converted via `exprToProtoInternal` - [ ] Return type correctly serialized - [ ] `getSupportLevel` reflects true compatibility: - `Compatible()` - matches Spark exactly - `Incompatible(Some("reason"))` - differs in documented ways - `Unsupported(Some("reason"))` - cannot be implemented - [ ] Serde in appropriate file (`datetime.scala`, `strings.scala`, `arithmetic.scala`, etc.) #### Registration (`QueryPlanSerde.scala`) - [ ] Added to correct map (temporal, string, arithmetic, etc.) - [ ] No duplicate registrations - [ ] Import statement added #### Rust Implementation (if applicable) Location: `native/spark-expr/src/` - [ ] Matches DataFusion and Arrow conventions - [ ] Null handling is correct - [ ] No panics. Use `Result` types. - [ ] Efficient array operations (avoid row-by-row) #### Tests - Prefer SQL File-Based Framework **Expression tests should use the SQL file-based framework (`CometSqlFileTestSuite`) where possible.** This framework automatically runs each query through both Spark and Comet and compares results. No Scala code is needed. Only fall back to Scala tests in `CometExpressionSuite` when the SQL framework cannot express the test. Examples include complex `DataFrame` setup, programmatic data generation, or non-expression tests. **SQL file test location:** `spark/src/test/resources/sql-tests/expressions//` Categories include: `aggregate/`, `array/`, `string/`, `math/`, `struct/`, `map/`, `datetime/`, `hash/`, etc. **SQL file structure:** ```sql -- Create test data statement CREATE TABLE test_crc32(col string, a int, b float) USING parquet statement INSERT INTO test_crc32 VALUES ('Spark', 10, 1.5), (NULL, NULL, NULL), ('', 0, 0.0) -- Default mode: verifies native Comet execution + result matches Spark query SELECT crc32(col) FROM test_crc32 -- spark_answer_only: compares results without requiring native execution query spark_answer_only SELECT crc32(cast(a as string)) FROM test_crc32 -- tolerance: allows numeric variance for floating-point results query tolerance=0.0001 SELECT cos(v) FROM test_trig -- expect_fallback: asserts fallback to Spark occurs query expect_fallback(unsupported expression) SELECT unsupported_func(v) FROM test_table -- expect_error: verifies both engines throw matching exceptions query expect_error(ARITHMETIC_OVERFLOW) SELECT 2147483647 + 1 -- ignore: skip queries with known bugs (include GitHub issue link) query ignore(https://github.com/apache/datafusion-comet/issues/NNNN) SELECT known_buggy_expr(v) FROM test_table ``` **Running SQL file tests:** ```bash # All SQL file tests ./mvnw test -Dsuites="org.apache.comet.CometSqlFileTestSuite" -Dtest=none # Specific test file (substring match) ./mvnw test -Dsuites="org.apache.comet.CometSqlFileTestSuite crc32" -Dtest=none ``` **CRITICAL: Verify all test requirements (regardless of framework):** - [ ] Basic functionality tested (column data, not just literals) - [ ] Null handling tested (`SELECT expression(NULL)`) - [ ] Edge cases tested (empty input, overflow, boundary values) - [ ] Both literal values and column references tested (they use different code paths) - [ ] For timestamp/datetime expressions, timezone handling is tested (e.g., UTC, non-UTC session timezone, timestamps with and without timezone) - [ ] One expression per SQL file for easier debugging - [ ] If using Scala tests instead, literal tests MUST disable constant folding: ```scala withSQLConf(SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> "org.apache.spark.sql.catalyst.optimizer.ConstantFolding") { checkSparkAnswerAndOperator("SELECT func(literal)") } ``` ### 5. Performance Review (Expression PRs) **For PRs that add new expressions, performance is not optional.** The whole point of Comet is to be faster than Spark. If a new expression is not faster, it may not be worth adding. 1. **Check that the PR includes microbenchmark results.** The PR description should contain benchmark numbers comparing Comet vs Spark for the new expression. If benchmark results are missing, flag this as a required addition. 2. **Look for a microbenchmark implementation.** Expression benchmarks live in `spark/src/test/scala/org/apache/spark/sql/benchmark/`. Check whether the PR adds a benchmark for the new expression. 3. **Review the benchmark results if provided:** - Is Comet actually faster than Spark for this expression? - Are the benchmarks representative? They should test with realistic data sizes, not just trivial inputs. - Are different data types benchmarked if the expression supports multiple types? 4. **Review the Rust implementation for performance concerns:** - Unnecessary allocations or copies - Row-by-row processing where batch/array operations are possible - Redundant type conversions - Inefficient string handling (e.g., repeated UTF-8 validation) - Missing use of Arrow compute kernels where they exist 5. **If benchmark results show Comet is slower than Spark**, flag this clearly. The PR should explain why the regression is acceptable or include a plan to optimize. ### 6. Check CI Test Failures **Always check the CI status and summarize any test failures in your review.** ```bash # View CI check status gh pr checks $ARGUMENTS --repo apache/datafusion-comet # View failed check details gh pr checks $ARGUMENTS --repo apache/datafusion-comet --failed ``` ### 7. Documentation Check Check whether the PR requires updates to user-facing documentation in `docs/`: - **Compatibility guide** (`docs/source/user-guide/compatibility.md`): New expressions or operators should be listed. Incompatible behaviors should be documented. - **Configuration guide** (`docs/source/user-guide/configs.md`): New config options should be documented. - **Expressions list** (`docs/source/user-guide/expressions.md`): New expressions should be added. If the PR adds a new expression or operator but does not update the relevant docs, flag this as something that needs to be addressed. ### 8. Common Comet Review Issues 1. **Incomplete type support**: Spark expression supports types not handled in PR 2. **Missing edge cases**: Null, overflow, empty string, negative values 3. **Wrong return type**: Return type must match Spark exactly 4. **Tests in wrong framework**: Expression tests should use the SQL file-based framework (`CometSqlFileTestSuite`) rather than adding to Scala test suites like `CometExpressionSuite`. Suggest migration if the PR adds Scala tests for expressions that could use SQL files instead. 5. **Stale native code**: PR might need `./mvnw install -pl common -DskipTests` 6. **Missing `getSupportLevel`**: Edge cases should be marked as `Incompatible` --- ## Output Format Present your review as guidance for the reviewer. Structure your output as: 1. **PR Summary** - Brief description of what the PR does 2. **CI Status** - Summary of CI check results 3. **Findings** - Your analysis organized by area (Spark compatibility, implementation, tests, etc.) 4. **Suggested Review Comments** - Specific comments the reviewer could leave on the PR, with file and line references where applicable ## Review Tone and Style Write reviews that sound human and conversational. Avoid: - Robotic or formulaic language - Em dashes. Use separate sentences instead. - Semicolons. Use separate sentences instead. Instead: - Write in flowing paragraphs using simple grammar - Keep sentences short and separate rather than joining them with punctuation - Be kind and constructive, even when raising concerns - Use backticks around any code references (function names, file paths, class names, types, config keys, etc.) - **Suggest** adding tests rather than stating tests are missing (e.g., "It might be worth adding a test for X" not "Tests are missing for X") - **Ask questions** about edge cases rather than asserting they aren't handled (e.g., "Does this handle the case where X is null?" not "This doesn't handle null") - Frame concerns as questions or suggestions when possible - Acknowledge what the PR does well before raising concerns ## Do Not Post Comments **IMPORTANT: Never post comments or reviews on the PR directly.** This skill is for providing guidance to a human reviewer. Present all findings and suggested comments to the user. The user will decide what to post. ================================================ FILE: .dockerignore ================================================ .git .github .idea bin conf docs/build docs/temp docs/venv metastore_db target common/target spark-integration/target fuzz-testing/target spark/target native/target core/target spark-warehouse venv ================================================ FILE: .github/ISSUE_TEMPLATE/bug_report.yml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. name: Bug report description: Create a bug report labels: bug body: - type: textarea attributes: label: Describe the bug description: Describe the bug. placeholder: > A clear and concise description of what the bug is. validations: required: true - type: textarea attributes: label: Steps to reproduce placeholder: > Describe steps to reproduce the bug: - type: textarea attributes: label: Expected behavior placeholder: > A clear and concise description of what you expected to happen. - type: textarea attributes: label: Additional context placeholder: > Add any other context about the problem here. ================================================ FILE: .github/ISSUE_TEMPLATE/feature_request.yml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. name: Feature request description: Suggest an idea for this project labels: enhancement body: - type: textarea attributes: label: What is the problem the feature request solves? description: Please describe how feature request improves the Comet. placeholder: > A clear and concise description of what the improvement is. Ex. I'm always frustrated when [...] (This section helps Comet developers understand the context and *why* for this feature, in addition to the *what*) - type: textarea attributes: label: Describe the potential solution placeholder: > A clear and concise description of what you want to happen. - type: textarea attributes: label: Additional context placeholder: > Add any other context or screenshots about the feature request here. ================================================ FILE: .github/actions/java-test/action.yaml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. name: "Java Test" description: "Run Java tests" inputs: artifact_name: description: "Unique name for uploaded artifacts for this run" required: true suites: description: 'Which Scalatest test suites to run' required: false default: '' maven_opts: description: 'Maven options passed to the mvn command' required: false default: '' scan_impl: description: 'The default Parquet scan implementation' required: false default: 'auto' upload-test-reports: description: 'Whether to upload test results including coverage to GitHub' required: false default: 'false' skip-native-build: description: 'Skip native build (when using pre-built artifact)' required: false default: 'false' runs: using: "composite" steps: - name: Run Cargo release build if: ${{ inputs.skip-native-build != 'true' }} shell: bash # it is important that we run the Scala tests against a release build rather than a debug build # to make sure that no tests are relying on overflow checks that are present only in debug builds run: | cd native cargo build --release - name: Cache Maven dependencies # TODO: remove next line after working again # temporarily work around https://github.com/actions/runner-images/issues/13341 # by disabling caching for macOS if: ${{ runner.os != 'macOS' }} uses: actions/cache@v5 with: path: | ~/.m2/repository /root/.m2/repository key: ${{ runner.os }}-java-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-java-maven- - name: Run all tests shell: bash if: ${{ inputs.suites == '' }} env: COMET_PARQUET_SCAN_IMPL: ${{ inputs.scan_impl }} SPARK_LOCAL_HOSTNAME: "localhost" SPARK_LOCAL_IP: "127.0.0.1" run: | MAVEN_OPTS="-Xmx4G -Xms2G -XX:+UnlockDiagnosticVMOptions -XX:+ShowMessageBoxOnError -XX:+HeapDumpOnOutOfMemoryError -XX:ErrorFile=./hs_err_pid%p.log" SPARK_HOME=`pwd` ./mvnw -B -Prelease install ${{ inputs.maven_opts }} - name: Run specified tests shell: bash if: ${{ inputs.suites != '' }} env: COMET_PARQUET_SCAN_IMPL: ${{ inputs.scan_impl }} SPARK_LOCAL_HOSTNAME: "localhost" SPARK_LOCAL_IP: "127.0.0.1" run: | MAVEN_SUITES="$(echo "${{ inputs.suites }}" | paste -sd, -)" echo "Running with MAVEN_SUITES=$MAVEN_SUITES" MAVEN_OPTS="-Xmx4G -Xms2G -DwildcardSuites=$MAVEN_SUITES -XX:+UnlockDiagnosticVMOptions -XX:+ShowMessageBoxOnError -XX:+HeapDumpOnOutOfMemoryError -XX:ErrorFile=./hs_err_pid%p.log" SPARK_HOME=`pwd` ./mvnw -B -Prelease install ${{ inputs.maven_opts }} - name: Upload crash logs if: failure() uses: actions/upload-artifact@v6 with: name: crash-logs-${{ inputs.artifact_name }} path: "**/hs_err_pid*.log" - name: Debug listing if: failure() shell: bash run: | echo "CWD: $(pwd)" ls -lah . ls -lah target find . -name 'unit-tests.log' - name: Upload unit-tests.log if: failure() uses: actions/upload-artifact@v6 with: name: unit-tests-${{ inputs.artifact_name }} path: "**/target/unit-tests.log" - name: Upload test results if: ${{ inputs.upload-test-reports == 'true' }} uses: actions/upload-artifact@v6 with: name: java-test-reports-${{ inputs.artifact_name }} path: "**/target/surefire-reports/*.txt" retention-days: 7 # 1 week for test reports overwrite: true ================================================ FILE: .github/actions/rust-test/action.yaml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. name: "Rust Test" description: "Run Rust tests" runs: using: "composite" steps: # Note: cargo fmt check is now handled by the lint job that gates this workflow - name: Check Cargo clippy shell: bash run: | cd native cargo clippy --color=never --all-targets --workspace -- -D warnings - name: Check compilation shell: bash run: | cd native cargo check --benches - name: Check unused dependencies shell: bash run: | cd native cargo install cargo-machete --version 0.7.0 && cargo machete - name: Cache Maven dependencies uses: actions/cache@v4 with: path: | ~/.m2/repository /root/.m2/repository key: ${{ runner.os }}-rust-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-rust-maven- - name: Build common module (pre-requisite for Rust tests) shell: bash run: | cd common ../mvnw -B clean compile -DskipTests - name: Install nextest shell: bash run: | cargo install cargo-nextest --locked - name: Run Cargo test shell: bash run: | cd native # Set LD_LIBRARY_PATH to include JVM library path for tests that use JNI export LD_LIBRARY_PATH=${JAVA_HOME}/lib/server:${LD_LIBRARY_PATH} RUST_BACKTRACE=1 cargo nextest run ================================================ FILE: .github/actions/setup-builder/action.yaml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. name: Prepare Builder description: 'Prepare Build Environment' inputs: rust-version: description: 'version of rust to install (e.g. nightly)' required: true default: 'stable' jdk-version: description: 'jdk version to install (e.g., 17)' required: true default: '17' runs: using: "composite" steps: - name: Install Build Dependencies shell: bash run: | apt-get update apt-get install -y protobuf-compiler apt-get install -y clang - name: Install JDK ${{inputs.jdk-version}} uses: actions/setup-java@v4 with: # distribution is chosen to be zulu as it still offers JDK 8 with Silicon support, which # is not available in the adopt distribution distribution: 'zulu' java-version: ${{inputs.jdk-version}} - name: Set JAVA_HOME shell: bash run: echo "JAVA_HOME=$(echo ${JAVA_HOME})" >> $GITHUB_ENV - name: Setup Rust toolchain shell: bash # rustfmt is needed for the substrait build script run: | echo "Installing ${{inputs.rust-version}}" rustup toolchain install ${{inputs.rust-version}} rustup default ${{inputs.rust-version}} rustup component add rustfmt clippy ================================================ FILE: .github/actions/setup-iceberg-builder/action.yaml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. name: Setup Iceberg Builder description: 'Setup Apache Iceberg to run Spark SQL tests' inputs: iceberg-version: description: 'The Apache Iceberg version (e.g., 1.8.1) to build' required: true runs: using: "composite" steps: - name: Clone Iceberg repo uses: actions/checkout@v6 with: repository: apache/iceberg path: apache-iceberg ref: apache-iceberg-${{inputs.iceberg-version}} fetch-depth: 1 - name: Setup Iceberg for Comet shell: bash run: | cd apache-iceberg git apply ../dev/diffs/iceberg/${{inputs.iceberg-version}}.diff ================================================ FILE: .github/actions/setup-macos-builder/action.yaml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. name: Prepare Builder for MacOS description: 'Prepare Build Environment' inputs: rust-version: description: 'version of rust to install (e.g. nightly)' required: true default: 'stable' jdk-version: description: 'jdk version to install (e.g., 17)' required: true default: '17' jdk-architecture: description: 'OS architecture for the JDK' required: true default: 'x64' protoc-architecture: description: 'OS architecture for protobuf compiler' required: true default: 'x86_64' runs: using: "composite" steps: - name: Install Build Dependencies shell: bash run: | # Install protobuf mkdir -p $HOME/d/protoc cd $HOME/d/protoc export PROTO_ZIP="protoc-21.4-osx-${{inputs.protoc-architecture}}.zip" curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP unzip $PROTO_ZIP echo "$HOME/d/protoc/bin" >> $GITHUB_PATH export PATH=$PATH:$HOME/d/protoc/bin # install openssl and setup DYLD_LIBRARY_PATH brew install openssl OPENSSL_LIB_PATH=`brew --prefix openssl`/lib echo "openssl lib path is: ${OPENSSL_LIB_PATH}" echo "DYLD_LIBRARY_PATH=$OPENSSL_LIB_PATH:$DYLD_LIBRARY_PATH" >> $GITHUB_ENV # output the current status of SIP for later debugging csrutil status || true - name: Install JDK ${{inputs.jdk-version}} uses: actions/setup-java@v4 with: # distribution is chosen to be zulu as it still offers JDK 8 with Silicon support, which # is not available in the adopt distribution distribution: 'zulu' java-version: ${{inputs.jdk-version}} architecture: ${{inputs.jdk-architecture}} - name: Set JAVA_HOME shell: bash run: echo "JAVA_HOME=$(echo ${JAVA_HOME})" >> $GITHUB_ENV - name: Setup Rust toolchain shell: bash # rustfmt is needed for the substrait build script run: | echo "Installing ${{inputs.rust-version}}" rustup toolchain install ${{inputs.rust-version}} rustup default ${{inputs.rust-version}} rustup component add rustfmt clippy ================================================ FILE: .github/actions/setup-spark-builder/action.yaml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. name: Setup Spark Builder description: 'Setup Apache Spark to run SQL tests' inputs: spark-short-version: description: 'The Apache Spark short version (e.g., 3.5) to build' required: true spark-version: description: 'The Apache Spark version (e.g., 3.5.8) to build' required: true skip-native-build: description: 'Skip native build (when using pre-built artifact)' required: false default: 'false' runs: using: "composite" steps: - name: Clone Spark repo uses: actions/checkout@v6 with: repository: apache/spark path: apache-spark ref: v${{inputs.spark-version}} fetch-depth: 1 - name: Setup Spark for Comet shell: bash run: | cd apache-spark git apply ../dev/diffs/${{inputs.spark-version}}.diff - name: Cache Maven dependencies uses: actions/cache@v4 with: path: | ~/.m2/repository /root/.m2/repository key: ${{ runner.os }}-spark-sql-${{ hashFiles('spark/**/pom.xml', 'common/**/pom.xml') }} restore-keys: | ${{ runner.os }}-spark-sql- - name: Build Comet (with native) if: ${{ inputs.skip-native-build != 'true' }} shell: bash run: | PROFILES="-Pspark-${{inputs.spark-short-version}}" make release - name: Build Comet (Maven only, skip native) if: ${{ inputs.skip-native-build == 'true' }} shell: bash run: | # Native library should already be in native/target/release/ ./mvnw install -Prelease -DskipTests -Pspark-${{inputs.spark-short-version}} ================================================ FILE: .github/dependabot.yml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. version: 2 updates: - package-ecosystem: cargo directory: "/native" schedule: interval: weekly target-branch: main labels: [dependencies] ignore: # major version bumps of datafusion*, arrow*, parquet are handled manually - dependency-name: "arrow*" update-types: ["version-update:semver-major"] - dependency-name: "parquet" update-types: ["version-update:semver-major"] - dependency-name: "datafusion*" update-types: ["version-update:semver-major"] groups: proto: applies-to: version-updates patterns: - "prost*" - "pbjson*" # Catch-all: group only minor/patch into a single PR, # excluding deps we want always separate (and excluding arrow/parquet which have their own group) all-other-cargo-deps: applies-to: version-updates patterns: - "*" exclude-patterns: - "arrow*" - "parquet" - "object_store" - "sqlparser" - "prost*" - "pbjson*" update-types: - "minor" - "patch" - package-ecosystem: "github-actions" directory: "/" schedule: interval: "weekly" open-pull-requests-limit: 10 labels: [dependencies] ================================================ FILE: .github/pull_request_template.md ================================================ ## Which issue does this PR close? Closes #. ## Rationale for this change ## What changes are included in this PR? ## How are these changes tested? ================================================ FILE: .github/workflows/codeql.yml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # name: "CodeQL" on: push: branches: [ "main" ] pull_request: branches: [ "main" ] schedule: - cron: '16 4 * * 1' permissions: contents: read jobs: analyze: name: Analyze Actions runs-on: ubuntu-latest permissions: contents: read security-events: write packages: read steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: persist-credentials: false - name: Initialize CodeQL uses: github/codeql-action/init@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4 with: languages: actions - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4 with: category: "/language:actions" ================================================ FILE: .github/workflows/docker-publish.yml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. name: Publish Docker images concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} cancel-in-progress: true on: push: tags: - '*.*.*' - '*.*.*-rc*' - 'test-docker-publish-*' jobs: docker: name: Docker if: ${{ startsWith(github.repository, 'apache/') }} runs-on: ubuntu-22.04 permissions: contents: read packages: write steps: - name: Remove unnecessary files run: | echo "Disk space before cleanup:" df -h docker system prune -af sudo rm -rf /tmp/* sudo rm -rf /opt/hostedtoolcache sudo rm -rf "$AGENT_TOOLSDIRECTORY" sudo apt-get clean echo "Disk space after cleanup:" df -h - name: Set up Java uses: actions/setup-java@v5 with: java-version: '17' distribution: 'temurin' cache: 'maven' - name: Extract Comet version id: extract_version run: | # use the tag that triggered this workflow as the Comet version e.g. 0.4.0-rc1 echo "COMET_VERSION=${GITHUB_REF##*/}" >> $GITHUB_ENV - name: Echo Comet version run: echo "The current Comet version is ${{ env.COMET_VERSION }}" - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 - name: Login to GitHub Container Registry uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push uses: docker/build-push-action@v7 with: platforms: linux/amd64,linux/arm64 push: true tags: ghcr.io/apache/datafusion-comet:spark-3.5-scala-2.12-${{ env.COMET_VERSION }} file: kube/Dockerfile no-cache: true ================================================ FILE: .github/workflows/docs.yaml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. on: push: branches: - main paths: - .asf.yaml - .github/workflows/docs.yaml - docs/** name: Deploy DataFusion Comet site jobs: build-docs: name: Build docs if: ${{ startsWith(github.repository, 'apache/') }} runs-on: ubuntu-latest steps: - name: Checkout docs sources uses: actions/checkout@v6 - name: Checkout asf-site branch uses: actions/checkout@v6 with: ref: asf-site path: asf-site - name: Setup Python uses: actions/setup-python@v6 with: python-version: "3.10" - name: Setup Java uses: actions/setup-java@v5 with: distribution: 'temurin' java-version: '17' cache: 'maven' - name: Install dependencies run: | set -x python3 -m venv venv source venv/bin/activate pip install -r docs/requirements.txt - name: Build docs run: | set -x source venv/bin/activate cd docs ./build.sh - name: Copy & push the generated HTML run: | set -x cd asf-site/ rsync \ -a \ --delete \ --exclude '/.git/' \ ../docs/build/html/ \ ./ cp ../.asf.yaml . touch .nojekyll git status --porcelain if [ "$(git status --porcelain)" != "" ]; then git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" git add --all git commit -m 'Publish built docs triggered by ${{ github.sha }}' git push || git push --force fi ================================================ FILE: .github/workflows/iceberg_spark_test.yml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. name: Iceberg Spark SQL Tests concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} cancel-in-progress: true on: push: branches: - main paths-ignore: - "benchmarks/**" - "doc/**" - "docs/**" - "**.md" - "native/core/benches/**" - "native/spark-expr/benches/**" - "spark/src/test/scala/org/apache/spark/sql/benchmark/**" pull_request: paths-ignore: - "benchmarks/**" - "doc/**" - "docs/**" - "**.md" - "native/core/benches/**" - "native/spark-expr/benches/**" - "spark/src/test/scala/org/apache/spark/sql/benchmark/**" # manual trigger # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow workflow_dispatch: env: RUST_VERSION: stable RUST_BACKTRACE: 1 jobs: # Build native library once and share with all test jobs build-native: name: Build Native Library runs-on: ubuntu-24.04 container: image: amd64/rust steps: - uses: actions/checkout@v6 - name: Setup Rust & Java toolchain uses: ./.github/actions/setup-builder with: rust-version: ${{ env.RUST_VERSION }} jdk-version: 17 - name: Restore Cargo cache uses: actions/cache/restore@v5 with: path: | ~/.cargo/registry ~/.cargo/git native/target key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }} restore-keys: | ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}- - name: Build native library # Use CI profile for faster builds (no LTO) and to share cache with pr_build_linux.yml. run: | cd native && cargo build --profile ci env: RUSTFLAGS: "-Ctarget-cpu=x86-64-v3" - name: Save Cargo cache uses: actions/cache/save@v5 if: github.ref == 'refs/heads/main' with: path: | ~/.cargo/registry ~/.cargo/git native/target key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }} - name: Upload native library uses: actions/upload-artifact@v7 with: name: native-lib-iceberg path: native/target/ci/libcomet.so retention-days: 1 iceberg-spark: needs: build-native strategy: matrix: os: [ubuntu-24.04] java-version: [11, 17] iceberg-version: [{short: '1.8', full: '1.8.1'}, {short: '1.9', full: '1.9.1'}, {short: '1.10', full: '1.10.0'}] spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.8'}] scala-version: ['2.13'] fail-fast: false name: iceberg-spark/${{ matrix.os }}/iceberg-${{ matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{ matrix.scala-version }}/java-${{ matrix.java-version }} runs-on: ${{ matrix.os }} container: image: amd64/rust env: SPARK_LOCAL_IP: localhost steps: - uses: actions/checkout@v6 - name: Setup Rust & Java toolchain uses: ./.github/actions/setup-builder with: rust-version: ${{env.RUST_VERSION}} jdk-version: ${{ matrix.java-version }} - name: Download native library uses: actions/download-artifact@v8 with: name: native-lib-iceberg path: native/target/release/ - name: Build Comet run: | ./mvnw install -Prelease -DskipTests -Pspark-${{ matrix.spark-version.short }} -Pscala-${{ matrix.scala-version }} - name: Setup Iceberg uses: ./.github/actions/setup-iceberg-builder with: iceberg-version: ${{ matrix.iceberg-version.full }} - name: Run Iceberg Spark tests run: | cd apache-iceberg rm -rf /root/.m2/repository/org/apache/parquet # somehow parquet cache requires cleanups ENABLE_COMET=true ENABLE_COMET_ONHEAP=true ./gradlew -DsparkVersions=${{ matrix.spark-version.short }} -DscalaVersion=${{ matrix.scala-version }} -DflinkVersions= -DkafkaVersions= \ :iceberg-spark:iceberg-spark-${{ matrix.spark-version.short }}_${{ matrix.scala-version }}:test \ -Pquick=true -x javadoc iceberg-spark-extensions: needs: build-native strategy: matrix: os: [ubuntu-24.04] java-version: [11, 17] iceberg-version: [{short: '1.8', full: '1.8.1'}, {short: '1.9', full: '1.9.1'}, {short: '1.10', full: '1.10.0'}] spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.8'}] scala-version: ['2.13'] fail-fast: false name: iceberg-spark-extensions/${{ matrix.os }}/iceberg-${{ matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{ matrix.scala-version }}/java-${{ matrix.java-version }} runs-on: ${{ matrix.os }} container: image: amd64/rust env: SPARK_LOCAL_IP: localhost steps: - uses: actions/checkout@v6 - name: Setup Rust & Java toolchain uses: ./.github/actions/setup-builder with: rust-version: ${{env.RUST_VERSION}} jdk-version: ${{ matrix.java-version }} - name: Download native library uses: actions/download-artifact@v8 with: name: native-lib-iceberg path: native/target/release/ - name: Build Comet run: | ./mvnw install -Prelease -DskipTests -Pspark-${{ matrix.spark-version.short }} -Pscala-${{ matrix.scala-version }} - name: Setup Iceberg uses: ./.github/actions/setup-iceberg-builder with: iceberg-version: ${{ matrix.iceberg-version.full }} - name: Run Iceberg Spark extensions tests run: | cd apache-iceberg rm -rf /root/.m2/repository/org/apache/parquet # somehow parquet cache requires cleanups ENABLE_COMET=true ENABLE_COMET_ONHEAP=true ./gradlew -DsparkVersions=${{ matrix.spark-version.short }} -DscalaVersion=${{ matrix.scala-version }} -DflinkVersions= -DkafkaVersions= \ :iceberg-spark:iceberg-spark-extensions-${{ matrix.spark-version.short }}_${{ matrix.scala-version }}:test \ -Pquick=true -x javadoc iceberg-spark-runtime: needs: build-native strategy: matrix: os: [ubuntu-24.04] java-version: [11, 17] iceberg-version: [{short: '1.8', full: '1.8.1'}, {short: '1.9', full: '1.9.1'}, {short: '1.10', full: '1.10.0'}] spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.8'}] scala-version: ['2.13'] fail-fast: false name: iceberg-spark-runtime/${{ matrix.os }}/iceberg-${{ matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{ matrix.scala-version }}/java-${{ matrix.java-version }} runs-on: ${{ matrix.os }} container: image: amd64/rust env: SPARK_LOCAL_IP: localhost steps: - uses: actions/checkout@v6 - name: Setup Rust & Java toolchain uses: ./.github/actions/setup-builder with: rust-version: ${{env.RUST_VERSION}} jdk-version: ${{ matrix.java-version }} - name: Download native library uses: actions/download-artifact@v8 with: name: native-lib-iceberg path: native/target/release/ - name: Build Comet run: | ./mvnw install -Prelease -DskipTests -Pspark-${{ matrix.spark-version.short }} -Pscala-${{ matrix.scala-version }} - name: Setup Iceberg uses: ./.github/actions/setup-iceberg-builder with: iceberg-version: ${{ matrix.iceberg-version.full }} - name: Run Iceberg Spark runtime tests run: | cd apache-iceberg rm -rf /root/.m2/repository/org/apache/parquet # somehow parquet cache requires cleanups ENABLE_COMET=true ENABLE_COMET_ONHEAP=true ./gradlew -DsparkVersions=${{ matrix.spark-version.short }} -DscalaVersion=${{ matrix.scala-version }} -DflinkVersions= -DkafkaVersions= \ :iceberg-spark:iceberg-spark-runtime-${{ matrix.spark-version.short }}_${{ matrix.scala-version }}:integrationTest \ -Pquick=true -x javadoc ================================================ FILE: .github/workflows/label_new_issues.yml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. name: Label new issues with requires-triage on: issues: types: [opened] permissions: issues: write jobs: add-triage-label: runs-on: ubuntu-latest steps: - uses: actions/github-script@v9 with: script: | await github.rest.issues.addLabels({ owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number, labels: ['requires-triage'] }) ================================================ FILE: .github/workflows/miri.yml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. name: Run Miri Safety Checks on: push: branches: - main paths-ignore: - "doc/**" - "docs/**" - "**.md" - "native/core/benches/**" - "native/spark-expr/benches/**" - "spark/src/test/scala/org/apache/spark/sql/benchmark/**" pull_request: paths-ignore: - "doc/**" - "docs/**" - "**.md" - "native/core/benches/**" - "native/spark-expr/benches/**" - "spark/src/test/scala/org/apache/spark/sql/benchmark/**" # manual trigger # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow workflow_dispatch: jobs: miri: name: "Miri" runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - name: Install Build Dependencies shell: bash run: | sudo apt-get update sudo apt-get install -y protobuf-compiler sudo apt-get install -y clang - name: Install Miri run: | rustup toolchain install nightly --component miri rustup override set nightly cargo miri setup - name: Test with Miri run: | cd native MIRIFLAGS="-Zmiri-disable-isolation" cargo miri test --lib --bins --tests --examples ================================================ FILE: .github/workflows/pr_benchmark_check.yml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # Lightweight CI for benchmark-only changes - verifies compilation and linting # without running full test suites name: PR Benchmark Check concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} cancel-in-progress: true on: push: branches: - main paths: - "native/core/benches/**" - "native/spark-expr/benches/**" - "spark/src/test/scala/org/apache/spark/sql/benchmark/**" pull_request: paths: - "native/core/benches/**" - "native/spark-expr/benches/**" - "spark/src/test/scala/org/apache/spark/sql/benchmark/**" workflow_dispatch: env: RUST_VERSION: stable RUST_BACKTRACE: 1 jobs: benchmark-check: name: Benchmark Compile & Lint Check runs-on: ubuntu-latest container: image: amd64/rust steps: - uses: actions/checkout@v6 - name: Setup Rust & Java toolchain uses: ./.github/actions/setup-builder with: rust-version: ${{ env.RUST_VERSION }} jdk-version: 17 - name: Check Cargo fmt run: | cd native cargo fmt --all -- --check --color=never - name: Check Cargo clippy run: | cd native cargo clippy --color=never --all-targets --workspace -- -D warnings - name: Check benchmark compilation run: | cd native cargo check --benches - name: Cache Maven dependencies uses: actions/cache@v5 with: path: | ~/.m2/repository /root/.m2/repository key: ${{ runner.os }}-benchmark-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-benchmark-maven- - name: Check Scala compilation and linting run: | ./mvnw -B compile test-compile scalafix:scalafix -Dscalafix.mode=CHECK -Psemanticdb -DskipTests ================================================ FILE: .github/workflows/pr_build_linux.yml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. name: PR Build (Linux) concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} cancel-in-progress: true on: push: branches: - main paths-ignore: - "benchmarks/**" - "doc/**" - "docs/**" - "**.md" - "native/core/benches/**" - "native/spark-expr/benches/**" - "spark/src/test/scala/org/apache/spark/sql/benchmark/**" pull_request: paths-ignore: - "benchmarks/**" - "doc/**" - "docs/**" - "**.md" - "native/core/benches/**" - "native/spark-expr/benches/**" - "spark/src/test/scala/org/apache/spark/sql/benchmark/**" # manual trigger # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow workflow_dispatch: env: RUST_VERSION: stable RUST_BACKTRACE: 1 jobs: # Fast lint check - gates all other jobs lint: name: Lint runs-on: ubuntu-latest container: image: amd64/rust steps: - uses: actions/checkout@v6 - name: Check Rust formatting run: | rustup component add rustfmt cd native && cargo fmt --all -- --check lint-java: needs: lint name: Lint Java (${{ matrix.profile.name }}) runs-on: ubuntu-latest container: image: amd64/rust env: JAVA_TOOL_OPTIONS: ${{ matrix.profile.java_version == '17' && '--add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED' || '' }} strategy: matrix: profile: - name: "Spark 3.4, JDK 11, Scala 2.12" java_version: "11" maven_opts: "-Pspark-3.4 -Pscala-2.12" - name: "Spark 3.5, JDK 17, Scala 2.12" java_version: "17" maven_opts: "-Pspark-3.5 -Pscala-2.12" - name: "Spark 4.0, JDK 17" java_version: "17" maven_opts: "-Pspark-4.0" fail-fast: false steps: - uses: runs-on/action@742bf56072eb4845a0f94b3394673e4903c90ff0 # v2.1.0 - uses: actions/checkout@v6 - name: Setup Rust & Java toolchain uses: ./.github/actions/setup-builder with: rust-version: ${{ env.RUST_VERSION }} jdk-version: ${{ matrix.profile.java_version }} - name: Cache Maven dependencies uses: actions/cache@v5 with: path: | ~/.m2/repository /root/.m2/repository key: ${{ runner.os }}-java-maven-${{ hashFiles('**/pom.xml') }}-lint restore-keys: | ${{ runner.os }}-java-maven- - name: Run scalafix check run: | ./mvnw -B package -DskipTests scalafix:scalafix -Dscalafix.mode=CHECK -Psemanticdb ${{ matrix.profile.maven_opts }} - name: Setup Node.js uses: actions/setup-node@v6 with: node-version: '24' - name: Install prettier run: | npm install -g prettier - name: Run prettier run: | npx prettier "**/*.md" --write - name: Mark workspace as safe for git run: | git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: Check for any local git changes (such as generated docs) run: | ./dev/ci/check-working-tree-clean.sh # Build native library once and share with all test jobs build-native: needs: lint name: Build Native Library runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a+m7a+c8a,cpu=8,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion-comet', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust steps: - uses: runs-on/action@742bf56072eb4845a0f94b3394673e4903c90ff0 # v2.1.0 - uses: actions/checkout@v6 - name: Setup Rust toolchain uses: ./.github/actions/setup-builder with: rust-version: ${{ env.RUST_VERSION }} jdk-version: 17 # JDK only needed for common module proto generation - name: Restore Cargo cache uses: actions/cache/restore@v5 with: path: | ~/.cargo/registry ~/.cargo/git native/target key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }} restore-keys: | ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}- - name: Build native library (CI profile) run: | cd native # CI profile: same overflow behavior as release, but faster compilation # (no LTO, parallel codegen) cargo build --profile ci env: RUSTFLAGS: "-Ctarget-cpu=x86-64-v3" - name: Upload native library uses: actions/upload-artifact@v7 with: name: native-lib-linux path: native/target/ci/libcomet.so retention-days: 1 - name: Save Cargo cache uses: actions/cache/save@v5 if: github.ref == 'refs/heads/main' with: path: | ~/.cargo/registry ~/.cargo/git native/target key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }} # Run Rust tests (runs in parallel with build-native, uses debug builds) linux-test-rust: needs: lint name: ubuntu-latest/rust-test runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion-comet', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust steps: - uses: runs-on/action@742bf56072eb4845a0f94b3394673e4903c90ff0 # v2.1.0 - uses: actions/checkout@v6 - name: Setup Rust & Java toolchain uses: ./.github/actions/setup-builder with: rust-version: ${{ env.RUST_VERSION }} jdk-version: 17 - name: Restore Cargo cache uses: actions/cache/restore@v5 with: path: | ~/.cargo/registry ~/.cargo/git native/target # Note: Java version intentionally excluded - Rust target is JDK-independent key: ${{ runner.os }}-cargo-debug-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }} restore-keys: | ${{ runner.os }}-cargo-debug-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}- - name: Rust test steps uses: ./.github/actions/rust-test - name: Save Cargo cache uses: actions/cache/save@v5 if: github.ref == 'refs/heads/main' with: path: | ~/.cargo/registry ~/.cargo/git native/target key: ${{ runner.os }}-cargo-debug-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }} linux-test: needs: build-native strategy: matrix: # the goal with these profiles is to get coverage of all Java, Scala, and Spark # versions without testing all possible combinations, which would be overkill profile: - name: "Spark 3.4, JDK 11, Scala 2.12" java_version: "11" maven_opts: "-Pspark-3.4 -Pscala-2.12" scan_impl: "auto" - name: "Spark 3.5.5, JDK 17, Scala 2.13" java_version: "17" maven_opts: "-Pspark-3.5 -Dspark.version=3.5.5 -Pscala-2.13" scan_impl: "auto" - name: "Spark 3.5.6, JDK 17, Scala 2.13" java_version: "17" maven_opts: "-Pspark-3.5 -Dspark.version=3.5.6 -Pscala-2.13" scan_impl: "auto" - name: "Spark 3.5, JDK 17, Scala 2.12" java_version: "17" maven_opts: "-Pspark-3.5 -Pscala-2.12" scan_impl: "native_iceberg_compat" - name: "Spark 4.0, JDK 17" java_version: "17" maven_opts: "-Pspark-4.0" scan_impl: "auto" suite: - name: "fuzz" value: | org.apache.comet.CometFuzzTestSuite org.apache.comet.CometFuzzAggregateSuite org.apache.comet.CometFuzzIcebergSuite org.apache.comet.CometFuzzMathSuite org.apache.comet.DataGeneratorSuite - name: "shuffle" value: | org.apache.comet.exec.CometShuffleSuite org.apache.comet.exec.CometShuffle4_0Suite org.apache.comet.exec.CometNativeColumnarToRowSuite org.apache.comet.exec.CometNativeShuffleSuite org.apache.comet.exec.CometShuffleEncryptionSuite org.apache.comet.exec.CometShuffleManagerSuite org.apache.comet.exec.CometAsyncShuffleSuite org.apache.comet.exec.DisableAQECometShuffleSuite org.apache.comet.exec.DisableAQECometAsyncShuffleSuite org.apache.spark.shuffle.sort.SpillSorterSuite - name: "parquet" value: | org.apache.comet.parquet.CometParquetWriterSuite org.apache.comet.parquet.ParquetReadV1Suite org.apache.comet.parquet.ParquetReadV2Suite org.apache.comet.parquet.ParquetReadFromFakeHadoopFsSuite org.apache.spark.sql.comet.ParquetDatetimeRebaseV1Suite org.apache.spark.sql.comet.ParquetDatetimeRebaseV2Suite org.apache.spark.sql.comet.ParquetEncryptionITCase org.apache.comet.exec.CometNativeReaderSuite org.apache.comet.CometIcebergNativeSuite - name: "csv" value: | org.apache.comet.csv.CometCsvNativeReadSuite - name: "exec" value: | org.apache.comet.exec.CometAggregateSuite org.apache.comet.exec.CometExec3_4PlusSuite org.apache.comet.exec.CometExecSuite org.apache.comet.exec.CometGenerateExecSuite org.apache.comet.exec.CometWindowExecSuite org.apache.comet.exec.CometJoinSuite org.apache.comet.CometNativeSuite org.apache.comet.CometSparkSessionExtensionsSuite org.apache.spark.CometPluginsSuite org.apache.spark.CometPluginsDefaultSuite org.apache.spark.CometPluginsNonOverrideSuite org.apache.spark.CometPluginsUnifiedModeOverrideSuite org.apache.comet.rules.CometScanRuleSuite org.apache.comet.rules.CometExecRuleSuite org.apache.spark.sql.CometTPCDSQuerySuite org.apache.spark.sql.CometTPCDSQueryTestSuite org.apache.spark.sql.CometTPCHQuerySuite org.apache.spark.sql.comet.CometTPCDSV1_4_PlanStabilitySuite org.apache.spark.sql.comet.CometTPCDSV2_7_PlanStabilitySuite org.apache.spark.sql.comet.CometTaskMetricsSuite org.apache.spark.sql.comet.CometDppFallbackRepro3949Suite org.apache.spark.sql.comet.CometShuffleFallbackStickinessSuite org.apache.comet.objectstore.NativeConfigSuite - name: "expressions" value: | org.apache.comet.CometExpressionSuite org.apache.comet.CometSqlFileTestSuite org.apache.comet.CometExpressionCoverageSuite org.apache.comet.CometHashExpressionSuite org.apache.comet.CometTemporalExpressionSuite org.apache.comet.CometArrayExpressionSuite org.apache.comet.CometCastSuite org.apache.comet.CometDateTimeUtilsSuite org.apache.comet.CometMathExpressionSuite org.apache.comet.CometStringExpressionSuite org.apache.comet.CometBitwiseExpressionSuite org.apache.comet.CometMapExpressionSuite org.apache.comet.CometCsvExpressionSuite org.apache.comet.CometJsonExpressionSuite org.apache.comet.CometDateTimeUtilsSuite org.apache.comet.SparkErrorConverterSuite org.apache.comet.expressions.conditional.CometIfSuite org.apache.comet.expressions.conditional.CometCoalesceSuite org.apache.comet.expressions.conditional.CometCaseWhenSuite - name: "sql" value: | org.apache.spark.sql.CometToPrettyStringSuite fail-fast: false name: ${{ matrix.profile.name }}/${{ matrix.profile.scan_impl }} [${{ matrix.suite.name }}] runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion-comet', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust env: JAVA_TOOL_OPTIONS: ${{ matrix.profile.java_version == '17' && '--add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED' || '' }} steps: - uses: runs-on/action@742bf56072eb4845a0f94b3394673e4903c90ff0 # v2.1.0 - uses: actions/checkout@v6 - name: Setup Rust & Java toolchain uses: ./.github/actions/setup-builder with: rust-version: ${{ env.RUST_VERSION }} jdk-version: ${{ matrix.profile.java_version }} - name: Download native library uses: actions/download-artifact@v8 with: name: native-lib-linux # Download to release/ since Maven's -Prelease expects libcomet.so there path: native/target/release/ # Restore cargo registry cache (for any cargo commands that might run) - name: Cache Cargo registry uses: actions/cache@v5 with: path: | ~/.cargo/registry ~/.cargo/git key: ${{ runner.os }}-cargo-registry-${{ hashFiles('native/**/Cargo.lock') }} restore-keys: | ${{ runner.os }}-cargo-registry- - name: Java test steps uses: ./.github/actions/java-test with: artifact_name: ${{ matrix.profile.name }}-${{ matrix.suite.name }}-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}-${{ matrix.profile.scan_impl }} suites: ${{ matrix.suite.name == 'sql' && matrix.profile.name == 'Spark 3.4, JDK 11, Scala 2.12' && '' || matrix.suite.value }} maven_opts: ${{ matrix.profile.maven_opts }} scan_impl: ${{ matrix.profile.scan_impl }} upload-test-reports: true skip-native-build: true # TPC-H correctness test - verifies benchmark queries produce correct results verify-benchmark-results-tpch: needs: build-native name: Verify TPC-H Results runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion-comet', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust steps: - uses: runs-on/action@742bf56072eb4845a0f94b3394673e4903c90ff0 # v2.1.0 - uses: actions/checkout@v6 - name: Setup Rust & Java toolchain uses: ./.github/actions/setup-builder with: rust-version: ${{ env.RUST_VERSION }} jdk-version: 11 - name: Download native library uses: actions/download-artifact@v8 with: name: native-lib-linux path: native/target/release/ - name: Cache Maven dependencies uses: actions/cache@v5 with: path: | ~/.m2/repository /root/.m2/repository key: ${{ runner.os }}-java-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-java-maven- - name: Cache TPC-H data id: cache-tpch uses: actions/cache@v5 with: path: ./tpch key: tpch-${{ hashFiles('.github/workflows/pr_build_linux.yml') }} - name: Build project run: | ./mvnw -B -Prelease install -DskipTests - name: Generate TPC-H data (SF=1) if: steps.cache-tpch.outputs.cache-hit != 'true' run: | cd spark && MAVEN_OPTS='-Xmx20g' ../mvnw -B -Prelease exec:java -Dexec.mainClass="org.apache.spark.sql.GenTPCHData" -Dexec.classpathScope="test" -Dexec.cleanupDaemonThreads="false" -Dexec.args="--location `pwd`/.. --scaleFactor 1 --numPartitions 1 --overwrite" - name: Run TPC-H queries run: | SPARK_HOME=`pwd` SPARK_TPCH_DATA=`pwd`/tpch/sf1_parquet ./mvnw -B -Prelease -Dsuites=org.apache.spark.sql.CometTPCHQuerySuite test # TPC-DS correctness tests - verifies benchmark queries produce correct results verify-benchmark-results-tpcds: needs: build-native name: Verify TPC-DS Results (${{ matrix.join }}) runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion-comet', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust strategy: matrix: join: [sort_merge, broadcast, hash] fail-fast: false steps: - uses: runs-on/action@742bf56072eb4845a0f94b3394673e4903c90ff0 # v2.1.0 - uses: actions/checkout@v6 - name: Setup Rust & Java toolchain uses: ./.github/actions/setup-builder with: rust-version: ${{ env.RUST_VERSION }} jdk-version: 11 - name: Download native library uses: actions/download-artifact@v8 with: name: native-lib-linux path: native/target/release/ - name: Cache Maven dependencies uses: actions/cache@v5 with: path: | ~/.m2/repository /root/.m2/repository key: ${{ runner.os }}-java-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-java-maven- - name: Cache TPC-DS data id: cache-tpcds uses: actions/cache@v5 with: path: ./tpcds-sf-1 key: tpcds-${{ hashFiles('.github/workflows/pr_build_linux.yml') }} - name: Build project run: | ./mvnw -B -Prelease install -DskipTests - name: Checkout tpcds-kit if: steps.cache-tpcds.outputs.cache-hit != 'true' uses: actions/checkout@v6 with: repository: databricks/tpcds-kit path: ./tpcds-kit - name: Build tpcds-kit if: steps.cache-tpcds.outputs.cache-hit != 'true' run: | apt-get update && apt-get install -y yacc bison flex gcc-12 g++-12 update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 120 --slave /usr/bin/g++ g++ /usr/bin/g++-12 cd tpcds-kit/tools && make OS=LINUX - name: Generate TPC-DS data (SF=1) if: steps.cache-tpcds.outputs.cache-hit != 'true' run: | cd spark && MAVEN_OPTS='-Xmx20g' ../mvnw -B -Prelease exec:java -Dexec.mainClass="org.apache.spark.sql.GenTPCDSData" -Dexec.classpathScope="test" -Dexec.cleanupDaemonThreads="false" -Dexec.args="--dsdgenDir `pwd`/../tpcds-kit/tools --location `pwd`/../tpcds-sf-1 --scaleFactor 1 --numPartitions 1" - name: Run TPC-DS queries (Sort merge join) if: matrix.join == 'sort_merge' run: | SPARK_HOME=`pwd` SPARK_TPCDS_DATA=`pwd`/tpcds-sf-1 ./mvnw -B -Prelease -Dsuites=org.apache.spark.sql.CometTPCDSQuerySuite test env: SPARK_TPCDS_JOIN_CONF: | spark.sql.autoBroadcastJoinThreshold=-1 spark.sql.join.preferSortMergeJoin=true - name: Run TPC-DS queries (Broadcast hash join) if: matrix.join == 'broadcast' run: | SPARK_HOME=`pwd` SPARK_TPCDS_DATA=`pwd`/tpcds-sf-1 ./mvnw -B -Prelease -Dsuites=org.apache.spark.sql.CometTPCDSQuerySuite test env: SPARK_TPCDS_JOIN_CONF: | spark.sql.autoBroadcastJoinThreshold=10485760 - name: Run TPC-DS queries (Shuffled hash join) if: matrix.join == 'hash' run: | SPARK_HOME=`pwd` SPARK_TPCDS_DATA=`pwd`/tpcds-sf-1 ./mvnw -B -Prelease -Dsuites=org.apache.spark.sql.CometTPCDSQuerySuite test env: SPARK_TPCDS_JOIN_CONF: | spark.sql.autoBroadcastJoinThreshold=-1 spark.sql.join.forceApplyShuffledHashJoin=true ================================================ FILE: .github/workflows/pr_build_macos.yml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. name: PR Build (macOS) concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} cancel-in-progress: true on: push: branches: - main paths-ignore: - "benchmarks/**" - "doc/**" - "docs/**" - "**.md" - "native/core/benches/**" - "native/spark-expr/benches/**" - "spark/src/test/scala/org/apache/spark/sql/benchmark/**" pull_request: paths-ignore: - "benchmarks/**" - "doc/**" - "docs/**" - "**.md" - "native/core/benches/**" - "native/spark-expr/benches/**" - "spark/src/test/scala/org/apache/spark/sql/benchmark/**" # manual trigger # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow workflow_dispatch: env: RUST_VERSION: stable RUST_BACKTRACE: 1 jobs: # Fast lint check - gates all other jobs (runs on Linux for cost efficiency) lint: name: Lint runs-on: ubuntu-latest container: image: amd64/rust steps: - uses: actions/checkout@v6 - name: Check Rust formatting run: | rustup component add rustfmt cd native && cargo fmt --all -- --check # Build native library once and share with all test jobs build-native: needs: lint name: Build Native Library (macOS) runs-on: macos-14 steps: - uses: actions/checkout@v6 - name: Setup Rust & Java toolchain uses: ./.github/actions/setup-macos-builder with: rust-version: ${{ env.RUST_VERSION }} jdk-version: 17 jdk-architecture: aarch64 protoc-architecture: aarch_64 - name: Restore Cargo cache uses: actions/cache/restore@v5 with: path: | ~/.cargo/registry ~/.cargo/git native/target key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }} restore-keys: | ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}- - name: Build native library (CI profile) run: | cd native # CI profile: same overflow behavior as release, but faster compilation # (no LTO, parallel codegen) cargo build --profile ci env: RUSTFLAGS: "-Ctarget-cpu=apple-m1" - name: Upload native library uses: actions/upload-artifact@v7 with: name: native-lib-macos path: native/target/ci/libcomet.dylib retention-days: 1 - name: Save Cargo cache uses: actions/cache/save@v5 if: github.ref == 'refs/heads/main' with: path: | ~/.cargo/registry ~/.cargo/git native/target key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }} macos-aarch64-test: needs: build-native strategy: matrix: os: [macos-14] # the goal with these profiles is to get coverage of all Java, Scala, and Spark # versions without testing all possible combinations, which would be overkill profile: - name: "Spark 3.4, JDK 11, Scala 2.12" java_version: "11" maven_opts: "-Pspark-3.4 -Pscala-2.12" - name: "Spark 3.5, JDK 17, Scala 2.13" java_version: "17" maven_opts: "-Pspark-3.5 -Pscala-2.13" - name: "Spark 4.0, JDK 17, Scala 2.13" java_version: "17" maven_opts: "-Pspark-4.0 -Pscala-2.13" suite: - name: "fuzz" value: | org.apache.comet.CometFuzzTestSuite org.apache.comet.CometFuzzAggregateSuite org.apache.comet.CometFuzzIcebergSuite org.apache.comet.CometFuzzMathSuite org.apache.comet.DataGeneratorSuite - name: "shuffle" value: | org.apache.comet.exec.CometShuffleSuite org.apache.comet.exec.CometShuffle4_0Suite org.apache.comet.exec.CometNativeColumnarToRowSuite org.apache.comet.exec.CometNativeShuffleSuite org.apache.comet.exec.CometShuffleEncryptionSuite org.apache.comet.exec.CometShuffleManagerSuite org.apache.comet.exec.CometAsyncShuffleSuite org.apache.comet.exec.DisableAQECometShuffleSuite org.apache.comet.exec.DisableAQECometAsyncShuffleSuite org.apache.spark.shuffle.sort.SpillSorterSuite - name: "parquet" value: | org.apache.comet.parquet.CometParquetWriterSuite org.apache.comet.parquet.ParquetReadV1Suite org.apache.comet.parquet.ParquetReadV2Suite org.apache.comet.parquet.ParquetReadFromFakeHadoopFsSuite org.apache.spark.sql.comet.ParquetDatetimeRebaseV1Suite org.apache.spark.sql.comet.ParquetDatetimeRebaseV2Suite org.apache.spark.sql.comet.ParquetEncryptionITCase org.apache.comet.exec.CometNativeReaderSuite org.apache.comet.CometIcebergNativeSuite - name: "csv" value: | org.apache.comet.csv.CometCsvNativeReadSuite - name: "exec" value: | org.apache.comet.exec.CometAggregateSuite org.apache.comet.exec.CometExec3_4PlusSuite org.apache.comet.exec.CometExecSuite org.apache.comet.exec.CometGenerateExecSuite org.apache.comet.exec.CometWindowExecSuite org.apache.comet.exec.CometJoinSuite org.apache.comet.CometNativeSuite org.apache.comet.CometSparkSessionExtensionsSuite org.apache.spark.CometPluginsSuite org.apache.spark.CometPluginsDefaultSuite org.apache.spark.CometPluginsNonOverrideSuite org.apache.spark.CometPluginsUnifiedModeOverrideSuite org.apache.comet.rules.CometScanRuleSuite org.apache.comet.rules.CometExecRuleSuite org.apache.spark.sql.CometTPCDSQuerySuite org.apache.spark.sql.CometTPCDSQueryTestSuite org.apache.spark.sql.CometTPCHQuerySuite org.apache.spark.sql.comet.CometTPCDSV1_4_PlanStabilitySuite org.apache.spark.sql.comet.CometTPCDSV2_7_PlanStabilitySuite org.apache.spark.sql.comet.CometTaskMetricsSuite org.apache.spark.sql.comet.CometDppFallbackRepro3949Suite org.apache.spark.sql.comet.CometShuffleFallbackStickinessSuite org.apache.comet.objectstore.NativeConfigSuite - name: "expressions" value: | org.apache.comet.CometExpressionSuite org.apache.comet.CometSqlFileTestSuite org.apache.comet.CometExpressionCoverageSuite org.apache.comet.CometHashExpressionSuite org.apache.comet.CometTemporalExpressionSuite org.apache.comet.CometArrayExpressionSuite org.apache.comet.CometCastSuite org.apache.comet.CometMathExpressionSuite org.apache.comet.CometStringExpressionSuite org.apache.comet.CometBitwiseExpressionSuite org.apache.comet.CometMapExpressionSuite org.apache.comet.CometJsonExpressionSuite org.apache.comet.CometCsvExpressionSuite org.apache.comet.CometDateTimeUtilsSuite org.apache.comet.SparkErrorConverterSuite org.apache.comet.expressions.conditional.CometIfSuite org.apache.comet.expressions.conditional.CometCoalesceSuite org.apache.comet.expressions.conditional.CometCaseWhenSuite - name: "sql" value: | org.apache.spark.sql.CometToPrettyStringSuite fail-fast: false name: ${{ matrix.os }}/${{ matrix.profile.name }} [${{ matrix.suite.name }}] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v6 - name: Setup Rust & Java toolchain uses: ./.github/actions/setup-macos-builder with: rust-version: ${{ env.RUST_VERSION }} jdk-version: ${{ matrix.profile.java_version }} jdk-architecture: aarch64 protoc-architecture: aarch_64 - name: Download native library uses: actions/download-artifact@v8 with: name: native-lib-macos # Download to release/ since Maven's -Prelease expects libcomet.dylib there path: native/target/release/ # Restore cargo registry cache (for any cargo commands that might run) - name: Cache Cargo registry uses: actions/cache@v5 with: path: | ~/.cargo/registry ~/.cargo/git key: ${{ runner.os }}-cargo-registry-${{ hashFiles('native/**/Cargo.lock') }} restore-keys: | ${{ runner.os }}-cargo-registry- - name: Set thread thresholds envs for spark test on macOS # see: https://github.com/apache/datafusion-comet/issues/2965 shell: bash run: | echo "SPARK_TEST_SQL_SHUFFLE_EXCHANGE_MAX_THREAD_THRESHOLD=256" >> $GITHUB_ENV echo "SPARK_TEST_SQL_RESULT_QUERY_STAGE_MAX_THREAD_THRESHOLD=256" >> $GITHUB_ENV echo "SPARK_TEST_HIVE_SHUFFLE_EXCHANGE_MAX_THREAD_THRESHOLD=48" >> $GITHUB_ENV echo "SPARK_TEST_HIVE_RESULT_QUERY_STAGE_MAX_THREAD_THRESHOLD=48" >> $GITHUB_ENV - name: Java test steps uses: ./.github/actions/java-test with: artifact_name: ${{ matrix.os }}-${{ matrix.profile.name }}-${{ matrix.suite.name }}-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }} suites: ${{ matrix.suite.name == 'sql' && matrix.profile.name == 'Spark 3.4, JDK 11, Scala 2.12' && '' || matrix.suite.value }} maven_opts: ${{ matrix.profile.maven_opts }} skip-native-build: true ================================================ FILE: .github/workflows/pr_markdown_format.yml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. name: Check Markdown Formatting concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} cancel-in-progress: true on: pull_request: paths: - '**.md' jobs: prettier-check: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - name: Setup Node.js uses: actions/setup-node@v6 with: node-version: '24' - name: Install prettier run: npm install -g prettier - name: Check markdown formatting run: | # if you encounter error, run prettier locally and commit changes using instructions at: # # https://datafusion.apache.org/comet/contributor-guide/development.html#how-to-format-md-document # prettier --check "**/*.md" ================================================ FILE: .github/workflows/pr_missing_suites.yml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. name: Check that all test suites are added to PR workflows on: push: branches: - main pull_request: types: [opened, synchronize, reopened] jobs: check-missing-suites: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - name: Check Missing Suites run: python3 dev/ci/check-suites.py ================================================ FILE: .github/workflows/pr_rat_check.yml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. name: RAT License Check concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} cancel-in-progress: true permissions: contents: read # No paths-ignore: this workflow must run for ALL changes including docs on: push: branches: - main pull_request: workflow_dispatch: jobs: rat-check: name: RAT License Check runs-on: ubuntu-slim steps: - uses: actions/checkout@v6 - name: Set up Java uses: actions/setup-java@v5 with: distribution: temurin java-version: 11 - name: Run RAT check run: ./mvnw -B -N apache-rat:check ================================================ FILE: .github/workflows/pr_title_check.yml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. name: Check PR Title concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} cancel-in-progress: true on: pull_request: types: [opened, edited, reopened] jobs: check-pr-title: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - name: Check PR title env: PR_TITLE: ${{ github.event.pull_request.title }} run: | if ! echo $PR_TITLE | grep -Eq '^(\w+)(\(.+\))?: .+$'; then echo "PR title does not follow conventional commit style." echo "Please use a title in the format: type: message, or type(scope): message" echo "Example: feat: Add support for sort-merge join" exit 1 fi ================================================ FILE: .github/workflows/spark_sql_test.yml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. name: Spark SQL Tests concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} cancel-in-progress: true on: push: branches: - main paths-ignore: - "benchmarks/**" - "doc/**" - "docs/**" - "**.md" - "native/core/benches/**" - "native/spark-expr/benches/**" - "spark/src/test/scala/org/apache/spark/sql/benchmark/**" pull_request: paths-ignore: - "benchmarks/**" - "doc/**" - "docs/**" - "**.md" - "native/core/benches/**" - "native/spark-expr/benches/**" - "spark/src/test/scala/org/apache/spark/sql/benchmark/**" # manual trigger # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow workflow_dispatch: inputs: collect-fallback-logs: description: 'Whether to collect Comet fallback reasons from spark sql unit test logs' required: false default: 'false' type: boolean env: RUST_VERSION: stable RUST_BACKTRACE: 1 jobs: # Build native library once and share with all test jobs build-native: name: Build Native Library runs-on: ubuntu-24.04 container: image: amd64/rust steps: - uses: actions/checkout@v6 - name: Setup Rust toolchain uses: ./.github/actions/setup-builder with: rust-version: ${{ env.RUST_VERSION }} jdk-version: 17 - name: Restore Cargo cache uses: actions/cache/restore@v5 with: path: | ~/.cargo/registry ~/.cargo/git native/target key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }} restore-keys: | ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}- - name: Build native library (CI profile) run: | cd native cargo build --profile ci env: RUSTFLAGS: "-Ctarget-cpu=x86-64-v3" - name: Upload native library uses: actions/upload-artifact@v7 with: name: native-lib-linux path: native/target/ci/libcomet.so retention-days: 1 - name: Save Cargo cache uses: actions/cache/save@v5 if: github.ref == 'refs/heads/main' with: path: | ~/.cargo/registry ~/.cargo/git native/target key: ${{ runner.os }}-cargo-ci-${{ hashFiles('native/**/Cargo.lock', 'native/**/Cargo.toml') }}-${{ hashFiles('native/**/*.rs') }} spark-sql-test: needs: build-native strategy: matrix: os: [ubuntu-24.04] module: - {name: "catalyst", args1: "catalyst/test", args2: ""} - {name: "sql_core-1", args1: "", args2: sql/testOnly * -- -l org.apache.spark.tags.ExtendedSQLTest -l org.apache.spark.tags.SlowSQLTest} - {name: "sql_core-2", args1: "", args2: "sql/testOnly * -- -n org.apache.spark.tags.ExtendedSQLTest"} - {name: "sql_core-3", args1: "", args2: "sql/testOnly * -- -n org.apache.spark.tags.SlowSQLTest"} - {name: "sql_hive-1", args1: "", args2: "hive/testOnly * -- -l org.apache.spark.tags.ExtendedHiveTest -l org.apache.spark.tags.SlowHiveTest"} - {name: "sql_hive-2", args1: "", args2: "hive/testOnly * -- -n org.apache.spark.tags.ExtendedHiveTest"} - {name: "sql_hive-3", args1: "", args2: "hive/testOnly * -- -n org.apache.spark.tags.SlowHiveTest"} # Since 4f5eaf0, auto mode uses native_datafusion for V1 scans, # so we only need to test with auto. config: - {spark-short: '3.4', spark-full: '3.4.3', java: 11, scan-impl: 'auto'} - {spark-short: '3.5', spark-full: '3.5.8', java: 11, scan-impl: 'auto'} - {spark-short: '4.0', spark-full: '4.0.1', java: 17, scan-impl: 'auto'} # Skip sql_hive-1 for Spark 4.0 due to https://github.com/apache/datafusion-comet/issues/2946 exclude: - config: {spark-short: '4.0', spark-full: '4.0.1', java: 17, scan-impl: 'auto'} module: {name: "sql_hive-1", args1: "", args2: "hive/testOnly * -- -l org.apache.spark.tags.ExtendedHiveTest -l org.apache.spark.tags.SlowHiveTest"} fail-fast: false name: spark-sql-${{ matrix.config.scan-impl }}-${{ matrix.module.name }}/spark-${{ matrix.config.spark-full }} runs-on: ${{ matrix.os }} container: image: amd64/rust steps: - uses: actions/checkout@v6 - name: Setup Rust & Java toolchain uses: ./.github/actions/setup-builder with: rust-version: ${{env.RUST_VERSION}} jdk-version: ${{ matrix.config.java }} - name: Download native library uses: actions/download-artifact@v8 with: name: native-lib-linux path: native/target/release/ - name: Setup Spark uses: ./.github/actions/setup-spark-builder with: spark-version: ${{ matrix.config.spark-full }} spark-short-version: ${{ matrix.config.spark-short }} skip-native-build: true - name: Run Spark tests run: | cd apache-spark rm -rf /root/.m2/repository/org/apache/parquet # somehow parquet cache requires cleanups NOLINT_ON_COMPILE=true ENABLE_COMET=true ENABLE_COMET_ONHEAP=true COMET_PARQUET_SCAN_IMPL=${{ matrix.config.scan-impl }} ENABLE_COMET_LOG_FALLBACK_REASONS=${{ github.event.inputs.collect-fallback-logs || 'false' }} \ build/sbt -Dsbt.log.noformat=true ${{ matrix.module.args1 }} "${{ matrix.module.args2 }}" if [ "${{ github.event.inputs.collect-fallback-logs }}" = "true" ]; then find . -type f -name "unit-tests.log" -print0 | xargs -0 grep -h "Comet cannot accelerate" | sed 's/.*Comet cannot accelerate/Comet cannot accelerate/' | sort -u > fallback.log fi env: LC_ALL: "C.UTF-8" - name: Upload fallback log if: ${{ github.event.inputs.collect-fallback-logs == 'true' }} uses: actions/upload-artifact@v7 with: name: fallback-log-spark-sql-${{ matrix.config.scan-impl }}-${{ matrix.module.name }}-spark-${{ matrix.config.spark-full }} path: "**/fallback.log" merge-fallback-logs: if: ${{ github.event.inputs.collect-fallback-logs == 'true' }} name: merge-fallback-logs needs: [spark-sql-test] runs-on: ubuntu-24.04 steps: - name: Download fallback log artifacts uses: actions/download-artifact@v8 with: path: fallback-logs/ - name: Merge fallback logs run: | find ./fallback-logs/ -type f -name "fallback.log" -print0 | xargs -0 cat | sort -u > all_fallback.log - name: Upload merged fallback log uses: actions/upload-artifact@v7 with: name: all-fallback-log path: all_fallback.log ================================================ FILE: .github/workflows/spark_sql_test_native_iceberg_compat.yml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. name: Spark SQL Tests (native_iceberg_compat) concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} cancel-in-progress: true on: # manual trigger # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow workflow_dispatch: env: RUST_VERSION: stable RUST_BACKTRACE: 1 jobs: spark-sql-catalyst-native-iceberg-compat: strategy: matrix: os: [ubuntu-24.04] java-version: [11] spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.8'}] module: - {name: "catalyst", args1: "catalyst/test", args2: ""} - {name: "sql/core-1", args1: "", args2: sql/testOnly * -- -l org.apache.spark.tags.ExtendedSQLTest -l org.apache.spark.tags.SlowSQLTest} - {name: "sql/core-2", args1: "", args2: "sql/testOnly * -- -n org.apache.spark.tags.ExtendedSQLTest"} - {name: "sql/core-3", args1: "", args2: "sql/testOnly * -- -n org.apache.spark.tags.SlowSQLTest"} - {name: "sql/hive-1", args1: "", args2: "hive/testOnly * -- -l org.apache.spark.tags.ExtendedHiveTest -l org.apache.spark.tags.SlowHiveTest"} - {name: "sql/hive-2", args1: "", args2: "hive/testOnly * -- -n org.apache.spark.tags.ExtendedHiveTest"} - {name: "sql/hive-3", args1: "", args2: "hive/testOnly * -- -n org.apache.spark.tags.SlowHiveTest"} fail-fast: false name: spark-sql-native-iceberg-compat-${{ matrix.module.name }}/${{ matrix.os }}/spark-${{ matrix.spark-version.full }}/java-${{ matrix.java-version }} runs-on: ${{ matrix.os }} container: image: amd64/rust steps: - uses: actions/checkout@v6 - name: Setup Rust & Java toolchain uses: ./.github/actions/setup-builder with: rust-version: ${{env.RUST_VERSION}} jdk-version: ${{ matrix.java-version }} - name: Setup Spark uses: ./.github/actions/setup-spark-builder with: spark-version: ${{ matrix.spark-version.full }} spark-short-version: ${{ matrix.spark-version.short }} - name: Run Spark tests run: | cd apache-spark rm -rf /root/.m2/repository/org/apache/parquet # somehow parquet cache requires cleanups ENABLE_COMET=true ENABLE_COMET_ONHEAP=true COMET_PARQUET_SCAN_IMPL=native_iceberg_compat build/sbt -Dsbt.log.noformat=true ${{ matrix.module.args1 }} "${{ matrix.module.args2 }}" env: LC_ALL: "C.UTF-8" ================================================ FILE: .github/workflows/stale.yml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. name: "Close stale PRs" on: schedule: - cron: "30 1 * * *" jobs: close-stale-prs: runs-on: ubuntu-latest permissions: issues: write pull-requests: write steps: - uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0 with: stale-pr-message: "Thank you for your contribution. Unfortunately, this pull request is stale because it has been open 60 days with no activity. Please remove the stale label or comment or this will be closed in 7 days." days-before-pr-stale: 60 days-before-pr-close: 7 # do not close stale issues days-before-issue-stale: -1 days-before-issue-close: -1 repo-token: ${{ secrets.GITHUB_TOKEN }} ================================================ FILE: .github/workflows/take.yml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. name: Assign/unassign the issue via `take` or `untake` comment on: issue_comment: types: created permissions: issues: write jobs: issue_assign: runs-on: ubuntu-latest if: (!github.event.issue.pull_request) && (github.event.comment.body == 'take' || github.event.comment.body == 'untake') concurrency: group: ${{ github.actor }}-issue-assign steps: - name: Take or untake issue env: COMMENT_BODY: ${{ github.event.comment.body }} ISSUE_NUMBER: ${{ github.event.issue.number }} USER_LOGIN: ${{ github.event.comment.user.login }} REPO: ${{ github.repository }} TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | if [ "$COMMENT_BODY" == "take" ] then CODE=$(curl -H "Authorization: token $TOKEN" -LI https://api.github.com/repos/$REPO/issues/$ISSUE_NUMBER/assignees/$USER_LOGIN -o /dev/null -w '%{http_code}\n' -s) if [ "$CODE" -eq "204" ] then echo "Assigning issue $ISSUE_NUMBER to $USER_LOGIN" curl -X POST -H "Authorization: token $TOKEN" -H "Content-Type: application/json" -d "{\"assignees\": [\"$USER_LOGIN\"]}" https://api.github.com/repos/$REPO/issues/$ISSUE_NUMBER/assignees else echo "Cannot assign issue $ISSUE_NUMBER to $USER_LOGIN" fi elif [ "$COMMENT_BODY" == "untake" ] then echo "Unassigning issue $ISSUE_NUMBER from $USER_LOGIN" curl -X DELETE -H "Authorization: token $TOKEN" -H "Content-Type: application/json" -d "{\"assignees\": [\"$USER_LOGIN\"]}" https://api.github.com/repos/$REPO/issues/$ISSUE_NUMBER/assignees fi ================================================ FILE: .github/workflows/validate_workflows.yml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. name: Validate Github Workflows on: pull_request: paths: - ".github/workflows/*.yml" - ".github/workflows/*.yaml" push: branches: - main jobs: validate: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - name: Install actionlint run: | curl -sSfL https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash | bash echo "$PWD" >> $GITHUB_PATH - name: Lint GitHub Actions workflows run: actionlint -color --shellcheck=off ================================================ FILE: .gitignore ================================================ CLAUDE.md target .idea *.iml .vscode/ .bloop/ .metals/ derby.log metastore_db/ spark-warehouse/ dependency-reduced-pom.xml native/proto/src/generated prebuild .flattened-pom.xml rat.txt filtered_rat.txt dev/dist apache-rat-*.jar venv .venv dev/release/comet-rm/workdir spark/benchmarks .DS_Store comet-event-trace.json __pycache__ output ================================================ FILE: .mvn/wrapper/maven-wrapper.properties ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.6/apache-maven-3.9.6-bin.zip wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar ================================================ FILE: .scalafix.conf ================================================ // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. rules = [ ExplicitResultTypes, NoAutoTupling, RemoveUnused, DisableSyntax, LeakingImplicitClassVal, NoValInForComprehension, ProcedureSyntax, RedundantSyntax ] ================================================ FILE: CHANGELOG.md ================================================ # Apache DataFusion Comet Changelog Comprehensive changelogs for each release are available [here](dev/changelog). ================================================ FILE: LICENSE.txt ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -------------------------------------------------------------------------------- This project includes code from Apache Aurora. * dev/release/{release,changelog,release-candidate} are based on the scripts from Apache Aurora Copyright: 2016 The Apache Software Foundation. Home page: https://aurora.apache.org/ License: http://www.apache.org/licenses/LICENSE-2.0 ================================================ FILE: Makefile ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. .PHONY: all core jvm test clean release-linux release bench define spark_jvm_17_extra_args $(shell ./mvnw help:evaluate -q -DforceStdout -Dexpression=extraJavaTestArgs) endef # Build optional Comet native features (like hdfs e.g) FEATURES_ARG := $(shell ! [ -z $(COMET_FEATURES) ] && echo '--features=$(COMET_FEATURES)') all: core jvm core: cd native && cargo build $(FEATURES_ARG) test-rust: # We need to compile CometException so that the cargo test can pass ./mvnw compile -pl common -DskipTests $(PROFILES) cd native && cargo build $(FEATURES_ARG) && \ RUST_BACKTRACE=1 cargo test $(FEATURES_ARG) jvm: ./mvnw clean package -DskipTests $(PROFILES) test-jvm: core SPARK_HOME=`pwd` COMET_CONF_DIR=$(shell pwd)/conf RUST_BACKTRACE=1 ./mvnw verify $(PROFILES) test: test-rust test-jvm clean: cd native && cargo clean ./mvnw clean $(PROFILES) rm -rf .dist bench: cd native && RUSTFLAGS="-Ctarget-cpu=native" cargo bench $(FEATURES_ARG) $(filter-out $@,$(MAKECMDGOALS)) format: cd native && cargo fmt ./mvnw compile test-compile scalafix:scalafix -Psemanticdb $(PROFILES) ./mvnw spotless:apply $(PROFILES) # build native libs for amd64 architecture Linux/MacOS on a Linux/amd64 machine/container core-amd64-libs: cd native && RUSTFLAGS="-Ctarget-cpu=x86-64-v3" cargo build -j 2 --release $(FEATURES_ARG) ifdef HAS_OSXCROSS rustup target add x86_64-apple-darwin cd native && cargo build -j 2 --target x86_64-apple-darwin --release $(FEATURES_ARG) endif # build native libs for arm64 architecture Linux/MacOS on a Linux/arm64 machine/container core-arm64-libs: cd native && RUSTFLAGS="-Ctarget-cpu=neoverse-n1" cargo build -j 2 --release $(FEATURES_ARG) ifdef HAS_OSXCROSS rustup target add aarch64-apple-darwin cd native && cargo build -j 2 --target aarch64-apple-darwin --release $(FEATURES_ARG) endif core-amd64: rustup target add x86_64-apple-darwin cd native && RUSTFLAGS="-Ctarget-cpu=skylake" CC=o64-clang CXX=o64-clang++ cargo build --target x86_64-apple-darwin --release $(FEATURES_ARG) mkdir -p common/target/classes/org/apache/comet/darwin/x86_64 cp native/target/x86_64-apple-darwin/release/libcomet.dylib common/target/classes/org/apache/comet/darwin/x86_64 cd native && RUSTFLAGS="-Ctarget-cpu=x86-64-v3" cargo build --release $(FEATURES_ARG) mkdir -p common/target/classes/org/apache/comet/linux/amd64 cp native/target/release/libcomet.so common/target/classes/org/apache/comet/linux/amd64 jar -cf common/target/comet-native-x86_64.jar \ -C common/target/classes/org/apache/comet darwin \ -C common/target/classes/org/apache/comet linux ./dev/deploy-file common/target/comet-native-x86_64.jar comet-native-x86_64${COMET_CLASSIFIER} jar core-arm64: rustup target add aarch64-apple-darwin cd native && RUSTFLAGS="-Ctarget-cpu=apple-m1" CC=arm64-apple-darwin21.4-clang CXX=arm64-apple-darwin21.4-clang++ CARGO_FEATURE_NEON=1 cargo build --target aarch64-apple-darwin --release $(FEATURES_ARG) mkdir -p common/target/classes/org/apache/comet/darwin/aarch64 cp native/target/aarch64-apple-darwin/release/libcomet.dylib common/target/classes/org/apache/comet/darwin/aarch64 cd native && RUSTFLAGS="-Ctarget-cpu=neoverse-n1" cargo build --release $(FEATURES_ARG) mkdir -p common/target/classes/org/apache/comet/linux/aarch64 cp native/target/release/libcomet.so common/target/classes/org/apache/comet/linux/aarch64 jar -cf common/target/comet-native-aarch64.jar \ -C common/target/classes/org/apache/comet darwin \ -C common/target/classes/org/apache/comet linux ./dev/deploy-file common/target/comet-native-aarch64.jar comet-native-aarch64${COMET_CLASSIFIER} jar release-linux: clean rustup target add aarch64-apple-darwin x86_64-apple-darwin cd native && RUSTFLAGS="-Ctarget-cpu=apple-m1" CC=arm64-apple-darwin21.4-clang CXX=arm64-apple-darwin21.4-clang++ CARGO_FEATURE_NEON=1 cargo build --target aarch64-apple-darwin --release $(FEATURES_ARG) cd native && RUSTFLAGS="-Ctarget-cpu=skylake" CC=o64-clang CXX=o64-clang++ cargo build --target x86_64-apple-darwin --release $(FEATURES_ARG) cd native && RUSTFLAGS="-Ctarget-cpu=native" cargo build --release $(FEATURES_ARG) ./mvnw install -Prelease -DskipTests $(PROFILES) release: cd native && RUSTFLAGS="$(RUSTFLAGS) -Ctarget-cpu=native" cargo build --release $(FEATURES_ARG) ./mvnw install -Prelease -DskipTests $(PROFILES) release-nogit: cd native && RUSTFLAGS="-Ctarget-cpu=native" cargo build --release ./mvnw install -Prelease -DskipTests $(PROFILES) -Dmaven.gitcommitid.skip=true benchmark-%: release cd spark && COMET_CONF_DIR=$(shell pwd)/conf MAVEN_OPTS='-Xmx20g ${call spark_jvm_17_extra_args}' ../mvnw exec:java -Dexec.mainClass="$*" -Dexec.classpathScope="test" -Dexec.cleanupDaemonThreads="false" -Dexec.args="$(filter-out $@,$(MAKECMDGOALS))" $(PROFILES) .DEFAULT: @: # ignore arguments provided to benchmarks e.g. "make benchmark-foo -- --bar", we do not want to treat "--bar" as target ================================================ FILE: NOTICE.txt ================================================ Apache DataFusion Comet Copyright 2024 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). This product includes software developed at Apache Gluten (https://github.com/apache/incubator-gluten/) Specifically: - Optimizer rule to replace SortMergeJoin with ShuffleHashJoin This product includes software developed at DataFusion HDFS ObjectStore Contrib Package(https://github.com/datafusion-contrib/datafusion-objectstore-hdfs) This product includes software developed at DataFusion fs-hdfs3 Contrib Package(https://github.com/datafusion-contrib/fs-hdfs) ================================================ FILE: README.md ================================================ # Apache DataFusion Comet [![Apache licensed][license-badge]][license-url] [![Discord chat][discord-badge]][discord-url] [![Pending PRs][pending-pr-badge]][pending-pr-url] [![Maven Central][maven-badge]][maven-url] [license-badge]: https://img.shields.io/badge/license-Apache%20v2-blue.svg [license-url]: https://github.com/apache/datafusion-comet/blob/main/LICENSE.txt [discord-badge]: https://img.shields.io/discord/885562378132000778.svg?logo=discord&style=flat-square [discord-url]: https://discord.gg/3EAr4ZX6JK [pending-pr-badge]: https://img.shields.io/github/issues-search/apache/datafusion-comet?query=is%3Apr+is%3Aopen+draft%3Afalse+review%3Arequired+status%3Asuccess&label=Pending%20PRs&logo=github [pending-pr-url]: https://github.com/apache/datafusion-comet/pulls?q=is%3Apr+is%3Aopen+draft%3Afalse+review%3Arequired+status%3Asuccess+sort%3Aupdated-desc [maven-badge]: https://img.shields.io/maven-central/v/org.apache.datafusion/comet-spark-spark4.0_2.13 [maven-url]: https://search.maven.org/search?q=g:org.apache.datafusion%20AND%20comet-spark logo Apache DataFusion Comet is a high-performance accelerator for Apache Spark, built on top of the powerful [Apache DataFusion] query engine. Comet is designed to significantly enhance the performance of Apache Spark workloads while leveraging commodity hardware and seamlessly integrating with the Spark ecosystem without requiring any code changes. Comet also accelerates Apache Iceberg, when performing Parquet scans from Spark. [Apache DataFusion]: https://datafusion.apache.org # Benefits of Using Comet ## Run Spark Queries at DataFusion Speeds Comet delivers a performance speedup for many queries, enabling faster data processing and shorter time-to-insights. The following chart shows the time it takes to run the 22 TPC-H queries against 100 GB of data in Parquet format using a single executor with 8 cores. See the [Comet Benchmarking Guide](https://datafusion.apache.org/comet/contributor-guide/benchmarking.html) for details of the environment used for these benchmarks. When using Comet, the overall run time is reduced from 687 seconds to 302 seconds, a 2.2x speedup. ![](docs/source/_static/images/benchmark-results/0.11.0/tpch_allqueries.png) Here is a breakdown showing relative performance of Spark and Comet for each TPC-H query. ![](docs/source/_static/images/benchmark-results/0.11.0/tpch_queries_compare.png) The following charts shows how much Comet currently accelerates each query from the benchmark. ### Relative speedup ![](docs/source/_static/images/benchmark-results/0.11.0/tpch_queries_speedup_rel.png) ### Absolute speedup ![](docs/source/_static/images/benchmark-results/0.11.0/tpch_queries_speedup_abs.png) These benchmarks can be reproduced in any environment using the documentation in the [Comet Benchmarking Guide](https://datafusion.apache.org/comet/contributor-guide/benchmarking.html). We encourage you to run your own benchmarks. Results for our benchmark derived from TPC-DS are available in the [benchmarking guide](https://datafusion.apache.org/comet/contributor-guide/benchmark-results/tpc-ds.html). ## Use Commodity Hardware Comet leverages commodity hardware, eliminating the need for costly hardware upgrades or specialized hardware accelerators, such as GPUs or FPGA. By maximizing the utilization of commodity hardware, Comet ensures cost-effectiveness and scalability for your Spark deployments. ## Spark Compatibility Comet aims for 100% compatibility with all supported versions of Apache Spark, allowing you to integrate Comet into your existing Spark deployments and workflows seamlessly. With no code changes required, you can immediately harness the benefits of Comet's acceleration capabilities without disrupting your Spark applications. ## Tight Integration with Apache DataFusion Comet tightly integrates with the core Apache DataFusion project, leveraging its powerful execution engine. With seamless interoperability between Comet and DataFusion, you can achieve optimal performance and efficiency in your Spark workloads. ## Active Community Comet boasts a vibrant and active community of developers, contributors, and users dedicated to advancing the capabilities of Apache DataFusion and accelerating the performance of Apache Spark. ## Getting Started To get started with Apache DataFusion Comet, follow the [installation instructions](https://datafusion.apache.org/comet/user-guide/installation.html). Join the [DataFusion Slack and Discord channels](https://datafusion.apache.org/contributor-guide/communication.html) to connect with other users, ask questions, and share your experiences with Comet. Follow [Apache DataFusion Comet Overview](https://datafusion.apache.org/comet/about/index.html#comet-overview) to get more detailed information ## Contributing We welcome contributions from the community to help improve and enhance Apache DataFusion Comet. Whether it's fixing bugs, adding new features, writing documentation, or optimizing performance, your contributions are invaluable in shaping the future of Comet. Check out our [contributor guide](https://datafusion.apache.org/comet/contributor-guide/contributing.html) to get started. ## License Apache DataFusion Comet is licensed under the Apache License 2.0. See the [LICENSE.txt](LICENSE.txt) file for details. ## Acknowledgments We would like to express our gratitude to the Apache DataFusion community for their support and contributions to Comet. Together, we're building a faster, more efficient future for big data processing with Apache Spark. ================================================ FILE: benchmarks/Dockerfile ================================================ # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. FROM apache/datafusion-comet:0.7.0-spark3.5.5-scala2.12-java11 RUN apt update \ && apt install -y git python3 python3-pip \ && apt clean RUN cd /opt \ && git clone https://github.com/apache/datafusion-benchmarks.git ================================================ FILE: benchmarks/README.md ================================================ # Running Comet Benchmarks in Microk8s This guide explains how to run benchmarks derived from TPC-H and TPC-DS in Apache DataFusion Comet deployed in a local Microk8s cluster. ## Use Microk8s locally Install Micro8s following the instructions at https://microk8s.io/docs/getting-started and then perform these additional steps, ensuring that any existing kube config is backed up first. ```shell mkdir -p ~/.kube microk8s config > ~/.kube/config microk8s enable dns microk8s enable registry microk8s kubectl create serviceaccount spark ``` ## Build Comet Docker Image Run the following command from the root of this repository to build the Comet Docker image, or use a published Docker image from https://github.com/orgs/apache/packages?repo_name=datafusion-comet ```shell docker build -t apache/datafusion-comet -f kube/Dockerfile . ``` ## Build Comet Benchmark Docker Image Build the benchmark Docker image and push to the Microk8s Docker registry. ```shell docker build -t apache/datafusion-comet-tpcbench . docker tag apache/datafusion-comet-tpcbench localhost:32000/apache/datafusion-comet-tpcbench:latest docker push localhost:32000/apache/datafusion-comet-tpcbench:latest ``` ## Run benchmarks ```shell export SPARK_MASTER=k8s://https://127.0.0.1:16443 export COMET_DOCKER_IMAGE=localhost:32000/apache/datafusion-comet-tpcbench:latest # Location of Comet JAR within the Docker image export COMET_JAR=/opt/spark/jars/comet-spark-spark3.4_2.12-0.5.0-SNAPSHOT.jar $SPARK_HOME/bin/spark-submit \ --master $SPARK_MASTER \ --deploy-mode cluster \ --name comet-tpcbench \ --driver-memory 8G \ --conf spark.driver.memory=8G \ --conf spark.executor.instances=1 \ --conf spark.executor.memory=32G \ --conf spark.executor.cores=8 \ --conf spark.cores.max=8 \ --conf spark.task.cpus=1 \ --conf spark.executor.memoryOverhead=3G \ --jars local://$COMET_JAR \ --conf spark.executor.extraClassPath=$COMET_JAR \ --conf spark.driver.extraClassPath=$COMET_JAR \ --conf spark.plugins=org.apache.spark.CometPlugin \ --conf spark.sql.extensions=org.apache.comet.CometSparkSessionExtensions \ --conf spark.comet.enabled=true \ --conf spark.comet.exec.enabled=true \ --conf spark.comet.exec.all.enabled=true \ --conf spark.comet.cast.allowIncompatible=true \ --conf spark.comet.exec.shuffle.enabled=true \ --conf spark.comet.exec.shuffle.mode=auto \ --conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \ --conf spark.kubernetes.namespace=default \ --conf spark.kubernetes.driver.pod.name=tpcbench \ --conf spark.kubernetes.container.image=$COMET_DOCKER_IMAGE \ --conf spark.kubernetes.driver.volumes.hostPath.tpcdata.mount.path=/mnt/bigdata/tpcds/sf100/ \ --conf spark.kubernetes.driver.volumes.hostPath.tpcdata.options.path=/mnt/bigdata/tpcds/sf100/ \ --conf spark.kubernetes.executor.volumes.hostPath.tpcdata.mount.path=/mnt/bigdata/tpcds/sf100/ \ --conf spark.kubernetes.executor.volumes.hostPath.tpcdata.options.path=/mnt/bigdata/tpcds/sf100/ \ --conf spark.kubernetes.authenticate.caCertFile=/var/snap/microk8s/current/certs/ca.crt \ local:///opt/datafusion-benchmarks/runners/datafusion-comet/tpcbench.py \ --benchmark tpcds \ --data /mnt/bigdata/tpcds/sf100/ \ --queries /opt/datafusion-benchmarks/tpcds/queries-spark \ --iterations 1 ``` ================================================ FILE: benchmarks/pyspark/README.md ================================================ # PySpark Benchmarks A suite of PySpark benchmarks for comparing performance between Spark, Comet JVM, and Comet Native implementations. ## Available Benchmarks Run `python run_benchmark.py --list-benchmarks` to see all available benchmarks: - **shuffle-hash** - Shuffle all columns using hash partitioning on group_key - **shuffle-roundrobin** - Shuffle all columns using round-robin partitioning ## Prerequisites - Apache Spark cluster (standalone, YARN, or Kubernetes) - PySpark installed - Comet JAR built ## Build Comet JAR ```bash cd /path/to/datafusion-comet make release ``` ## Step 1: Generate Test Data Generate test data with realistic 50-column schema (nested structs, arrays, maps): ```bash spark-submit \ --master spark://master:7077 \ --executor-memory 16g \ generate_data.py \ --output /tmp/shuffle-benchmark-data \ --rows 10000000 \ --partitions 200 ``` ### Data Generation Options | Option | Default | Description | | -------------------- | ---------- | ---------------------------- | | `--output`, `-o` | (required) | Output path for parquet data | | `--rows`, `-r` | 10000000 | Number of rows | | `--partitions`, `-p` | 200 | Number of output partitions | ## Step 2: Run Benchmarks ### List Available Benchmarks ```bash python run_benchmark.py --list-benchmarks ``` ### Run Individual Benchmarks You can run specific benchmarks by name: ```bash # Hash partitioning shuffle - Spark baseline spark-submit --master spark://master:7077 \ run_benchmark.py --data /tmp/shuffle-benchmark-data --mode spark --benchmark shuffle-hash # Round-robin shuffle - Spark baseline spark-submit --master spark://master:7077 \ run_benchmark.py --data /tmp/shuffle-benchmark-data --mode spark --benchmark shuffle-roundrobin # Hash partitioning - Comet JVM shuffle spark-submit --master spark://master:7077 \ --jars /path/to/comet.jar \ --conf spark.comet.enabled=true \ --conf spark.comet.exec.shuffle.enabled=true \ --conf spark.comet.shuffle.mode=jvm \ --conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \ run_benchmark.py --data /tmp/shuffle-benchmark-data --mode jvm --benchmark shuffle-hash # Round-robin - Comet Native shuffle spark-submit --master spark://master:7077 \ --jars /path/to/comet.jar \ --conf spark.comet.enabled=true \ --conf spark.comet.exec.shuffle.enabled=true \ --conf spark.comet.exec.shuffle.mode=native \ --conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \ run_benchmark.py --data /tmp/shuffle-benchmark-data --mode native --benchmark shuffle-roundrobin ``` ### Run All Benchmarks Use the provided script to run all benchmarks across all modes: ```bash SPARK_MASTER=spark://master:7077 \ EXECUTOR_MEMORY=16g \ ./run_all_benchmarks.sh /tmp/shuffle-benchmark-data ``` ## Checking Results Open the Spark UI (default: http://localhost:4040) during each benchmark run to compare shuffle write sizes in the Stages tab. ## Adding New Benchmarks The benchmark framework makes it easy to add new benchmarks: 1. **Create a benchmark class** in `benchmarks/` directory (or add to existing file): ```python from benchmarks.base import Benchmark class MyBenchmark(Benchmark): @classmethod def name(cls) -> str: return "my-benchmark" @classmethod def description(cls) -> str: return "Description of what this benchmark does" def run(self) -> Dict[str, Any]: # Read data df = self.spark.read.parquet(self.data_path) # Run your benchmark operation def benchmark_operation(): result = df.filter(...).groupBy(...).agg(...) result.write.mode("overwrite").parquet("/tmp/output") # Time it duration_ms = self._time_operation(benchmark_operation) return { 'duration_ms': duration_ms, # Add any other metrics you want to track } ``` 2. **Register the benchmark** in `benchmarks/__init__.py`: ```python from .my_module import MyBenchmark _BENCHMARK_REGISTRY = { # ... existing benchmarks MyBenchmark.name(): MyBenchmark, } ``` 3. **Run your new benchmark**: ```bash python run_benchmark.py --data /path/to/data --mode spark --benchmark my-benchmark ``` The base `Benchmark` class provides: - Automatic timing via `_time_operation()` - Standard output formatting via `execute_timed()` - Access to SparkSession, data path, and mode - Spark configuration printing ================================================ FILE: benchmarks/pyspark/benchmarks/__init__.py ================================================ #!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. """ Benchmark registry for PySpark benchmarks. This module provides a central registry for discovering and running benchmarks. """ from typing import Dict, Type, List from .base import Benchmark from .shuffle import ShuffleHashBenchmark, ShuffleRoundRobinBenchmark # Registry of all available benchmarks _BENCHMARK_REGISTRY: Dict[str, Type[Benchmark]] = { ShuffleHashBenchmark.name(): ShuffleHashBenchmark, ShuffleRoundRobinBenchmark.name(): ShuffleRoundRobinBenchmark, } def get_benchmark(name: str) -> Type[Benchmark]: """ Get a benchmark class by name. Args: name: Benchmark name Returns: Benchmark class Raises: KeyError: If benchmark name is not found """ if name not in _BENCHMARK_REGISTRY: available = ", ".join(sorted(_BENCHMARK_REGISTRY.keys())) raise KeyError( f"Unknown benchmark: {name}. Available benchmarks: {available}" ) return _BENCHMARK_REGISTRY[name] def list_benchmarks() -> List[tuple[str, str]]: """ List all available benchmarks. Returns: List of (name, description) tuples """ benchmarks = [] for name in sorted(_BENCHMARK_REGISTRY.keys()): benchmark_cls = _BENCHMARK_REGISTRY[name] benchmarks.append((name, benchmark_cls.description())) return benchmarks __all__ = [ 'Benchmark', 'get_benchmark', 'list_benchmarks', 'ShuffleHashBenchmark', 'ShuffleRoundRobinBenchmark', ] ================================================ FILE: benchmarks/pyspark/benchmarks/base.py ================================================ #!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. """ Base benchmark class providing common functionality for all benchmarks. """ import time from abc import ABC, abstractmethod from typing import Dict, Any from pyspark.sql import SparkSession class Benchmark(ABC): """Base class for all PySpark benchmarks.""" def __init__(self, spark: SparkSession, data_path: str, mode: str): """ Initialize benchmark. Args: spark: SparkSession instance data_path: Path to input data mode: Execution mode (spark, jvm, native) """ self.spark = spark self.data_path = data_path self.mode = mode @classmethod @abstractmethod def name(cls) -> str: """Return the benchmark name (used for CLI).""" pass @classmethod @abstractmethod def description(cls) -> str: """Return a short description of the benchmark.""" pass @abstractmethod def run(self) -> Dict[str, Any]: """ Run the benchmark and return results. Returns: Dictionary containing benchmark results (must include 'duration_ms') """ pass def execute_timed(self) -> Dict[str, Any]: """ Execute the benchmark with timing and standard output. Returns: Dictionary containing benchmark results """ print(f"\n{'=' * 80}") print(f"Benchmark: {self.name()}") print(f"Mode: {self.mode.upper()}") print(f"{'=' * 80}") print(f"Data path: {self.data_path}") # Print relevant Spark configuration self._print_spark_config() # Clear cache before running self.spark.catalog.clearCache() # Run the benchmark print(f"\nRunning benchmark...") results = self.run() # Print results print(f"\nDuration: {results['duration_ms']:,} ms") if 'row_count' in results: print(f"Rows processed: {results['row_count']:,}") # Print any additional metrics for key, value in results.items(): if key not in ['duration_ms', 'row_count']: print(f"{key}: {value}") print(f"{'=' * 80}\n") return results def _print_spark_config(self): """Print relevant Spark configuration.""" conf = self.spark.sparkContext.getConf() print(f"Shuffle manager: {conf.get('spark.shuffle.manager', 'default')}") print(f"Comet enabled: {conf.get('spark.comet.enabled', 'false')}") print(f"Comet shuffle enabled: {conf.get('spark.comet.exec.shuffle.enabled', 'false')}") print(f"Comet shuffle mode: {conf.get('spark.comet.shuffle.mode', 'not set')}") print(f"Spark UI: {self.spark.sparkContext.uiWebUrl}") def _time_operation(self, operation_fn): """ Time an operation and return duration in milliseconds. Args: operation_fn: Function to time (takes no arguments) Returns: Duration in milliseconds """ start_time = time.time() operation_fn() duration_ms = int((time.time() - start_time) * 1000) return duration_ms ================================================ FILE: benchmarks/pyspark/benchmarks/shuffle.py ================================================ #!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. """ Shuffle benchmarks for comparing shuffle file sizes and performance. These benchmarks test different partitioning strategies (hash, round-robin) across Spark, Comet JVM, and Comet Native shuffle implementations. """ from typing import Dict, Any from pyspark.sql import DataFrame from .base import Benchmark class ShuffleBenchmark(Benchmark): """Base class for shuffle benchmarks with common repartitioning logic.""" def __init__(self, spark, data_path: str, mode: str, num_partitions: int = 200): """ Initialize shuffle benchmark. Args: spark: SparkSession instance data_path: Path to input parquet data mode: Execution mode (spark, jvm, native) num_partitions: Number of partitions to shuffle to """ super().__init__(spark, data_path, mode) self.num_partitions = num_partitions def _read_and_count(self) -> tuple[DataFrame, int]: """Read input data and count rows.""" df = self.spark.read.parquet(self.data_path) row_count = df.count() return df, row_count def _repartition(self, df: DataFrame) -> DataFrame: """ Repartition dataframe using the strategy defined by subclass. Args: df: Input dataframe Returns: Repartitioned dataframe """ raise NotImplementedError("Subclasses must implement _repartition") def _write_output(self, df: DataFrame, output_path: str): """Write repartitioned data to parquet.""" df.write.mode("overwrite").parquet(output_path) def run(self) -> Dict[str, Any]: """ Run the shuffle benchmark. Returns: Dictionary with duration_ms and row_count """ # Read input data df, row_count = self._read_and_count() print(f"Number of rows: {row_count:,}") # Define the benchmark operation def benchmark_operation(): # Repartition using the specific strategy repartitioned = self._repartition(df) # Write to parquet to force materialization output_path = f"/tmp/shuffle-benchmark-output-{self.mode}-{self.name()}" self._write_output(repartitioned, output_path) print(f"Wrote repartitioned data to: {output_path}") # Time the operation duration_ms = self._time_operation(benchmark_operation) return { 'duration_ms': duration_ms, 'row_count': row_count, 'num_partitions': self.num_partitions, } class ShuffleHashBenchmark(ShuffleBenchmark): """Shuffle benchmark using hash partitioning on a key column.""" @classmethod def name(cls) -> str: return "shuffle-hash" @classmethod def description(cls) -> str: return "Shuffle all columns using hash partitioning on group_key" def _repartition(self, df: DataFrame) -> DataFrame: """Repartition using hash partitioning on group_key.""" return df.repartition(self.num_partitions, "group_key") class ShuffleRoundRobinBenchmark(ShuffleBenchmark): """Shuffle benchmark using round-robin partitioning.""" @classmethod def name(cls) -> str: return "shuffle-roundrobin" @classmethod def description(cls) -> str: return "Shuffle all columns using round-robin partitioning" def _repartition(self, df: DataFrame) -> DataFrame: """Repartition using round-robin (no partition columns specified).""" return df.repartition(self.num_partitions) ================================================ FILE: benchmarks/pyspark/generate_data.py ================================================ #!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. """ Generate test data for shuffle size comparison benchmark. This script generates a parquet dataset with a realistic schema (100 columns including deeply nested structs, arrays, and maps) for benchmarking shuffle operations across Spark, Comet JVM, and Comet Native shuffle modes. """ import argparse from pyspark.sql import SparkSession from pyspark.sql import functions as F from pyspark.sql.types import ( StructType, StructField, IntegerType, LongType, DoubleType, StringType, BooleanType, DateType, TimestampType, ArrayType, MapType, DecimalType ) def generate_data(output_path: str, num_rows: int, num_partitions: int): """Generate test data with realistic schema and write to parquet.""" spark = SparkSession.builder \ .appName("ShuffleBenchmark-DataGen") \ .getOrCreate() print(f"Generating {num_rows:,} rows with {num_partitions} partitions") print(f"Output path: {output_path}") print("Schema: 100 columns including deeply nested structs, arrays, and maps") # Start with a range and build up the columns df = spark.range(0, num_rows, numPartitions=num_partitions) # Add columns using selectExpr for better performance df = df.selectExpr( # Key columns for grouping/partitioning (1-3) "cast(id % 1000 as int) as partition_key", "cast(id % 100 as int) as group_key", "id as row_id", # Integer columns (4-15) "cast(id % 10000 as int) as category_id", "cast(id % 500 as int) as region_id", "cast(id % 50 as int) as department_id", "cast((id * 7) % 1000000 as int) as customer_id", "cast((id * 13) % 100000 as int) as product_id", "cast(id % 12 + 1 as int) as month", "cast(id % 28 + 1 as int) as day", "cast(2020 + (id % 5) as int) as year", "cast((id * 17) % 256 as int) as priority", "cast((id * 19) % 1000 as int) as rank", "cast((id * 23) % 10000 as int) as score_int", "cast((id * 29) % 500 as int) as level", # Long columns (16-22) "id * 1000 as transaction_id", "(id * 17) % 10000000000 as account_number", "(id * 31) % 1000000000 as reference_id", "(id * 37) % 10000000000 as external_id", "(id * 41) % 1000000000 as correlation_id", "(id * 43) % 10000000000 as trace_id", "(id * 47) % 1000000000 as span_id", # Double columns (23-35) "cast(id % 10000 as double) / 100.0 as amount", "cast((id * 3) % 10000 as double) / 100.0 as price", "cast(id % 100 as double) / 100.0 as discount", "cast((id * 7) % 500 as double) / 10.0 as weight", "cast((id * 11) % 1000 as double) / 10.0 as height", "cast(id % 360 as double) as latitude", "cast((id * 2) % 360 as double) as longitude", "cast((id * 13) % 10000 as double) / 1000.0 as rate", "cast((id * 17) % 100 as double) / 100.0 as percentage", "cast((id * 19) % 1000 as double) as velocity", "cast((id * 23) % 500 as double) / 10.0 as acceleration", "cast((id * 29) % 10000 as double) / 100.0 as temperature", "cast((id * 31) % 1000 as double) / 10.0 as pressure", # String columns (36-50) "concat('user_', cast(id % 100000 as string)) as user_name", "concat('email_', cast(id % 50000 as string), '@example.com') as email", "concat('SKU-', lpad(cast(id % 10000 as string), 6, '0')) as sku", "concat('ORD-', cast(id as string)) as order_id", "array('pending', 'processing', 'shipped', 'delivered', 'cancelled')[cast(id % 5 as int)] as status", "array('USD', 'EUR', 'GBP', 'JPY', 'CAD')[cast(id % 5 as int)] as currency", "concat('Description for item ', cast(id % 1000 as string), ' with additional details') as description", "concat('REF-', lpad(cast(id % 100000 as string), 8, '0')) as reference_code", "concat('TXN-', cast(id as string), '-', cast(id % 1000 as string)) as transaction_code", "array('A', 'B', 'C', 'D', 'E')[cast(id % 5 as int)] as grade", "concat('Note: Record ', cast(id as string), ' processed successfully') as notes", "concat('Session-', lpad(cast(id % 10000 as string), 6, '0')) as session_id", "concat('Device-', cast(id % 1000 as string)) as device_id", "array('chrome', 'firefox', 'safari', 'edge')[cast(id % 4 as int)] as browser", "array('windows', 'macos', 'linux', 'ios', 'android')[cast(id % 5 as int)] as os", # Boolean columns (51-56) "id % 2 = 0 as is_active", "id % 3 = 0 as is_verified", "id % 7 = 0 as is_premium", "id % 5 = 0 as is_deleted", "id % 11 = 0 as is_featured", "id % 13 = 0 as is_archived", # Date and timestamp columns (57-60) "date_add(to_date('2020-01-01'), cast(id % 1500 as int)) as created_date", "date_add(to_date('2020-01-01'), cast((id + 30) % 1500 as int)) as updated_date", "date_add(to_date('2020-01-01'), cast((id + 60) % 1500 as int)) as expires_date", "to_timestamp(concat('2020-01-01 ', lpad(cast(id % 24 as string), 2, '0'), ':00:00')) as created_at", # Simple arrays (61-65) "array(cast(id % 100 as int), cast((id + 1) % 100 as int), cast((id + 2) % 100 as int), cast((id + 3) % 100 as int), cast((id + 4) % 100 as int)) as tag_ids", "array(cast(id % 1000 as double) / 10.0, cast((id * 2) % 1000 as double) / 10.0, cast((id * 3) % 1000 as double) / 10.0) as scores", "array(concat('tag_', cast(id % 20 as string)), concat('tag_', cast((id + 5) % 20 as string)), concat('tag_', cast((id + 10) % 20 as string))) as tags", "array(id % 2 = 0, id % 3 = 0, id % 5 = 0, id % 7 = 0) as flag_array", "array(id * 1000, id * 2000, id * 3000) as long_array", # Simple maps (66-68) "map('key1', cast(id % 100 as string), 'key2', cast((id * 2) % 100 as string), 'key3', cast((id * 3) % 100 as string)) as str_attributes", "map('score1', cast(id % 100 as double), 'score2', cast((id * 2) % 100 as double)) as double_attributes", "map(cast(id % 10 as int), concat('val_', cast(id % 100 as string)), cast((id + 1) % 10 as int), concat('val_', cast((id + 1) % 100 as string))) as int_key_map", # Level 2 nested struct: address with nested geo (69) "named_struct(" " 'street', concat(cast(id % 9999 as string), ' Main St')," " 'city', array('New York', 'Los Angeles', 'Chicago', 'Houston', 'Phoenix')[cast(id % 5 as int)]," " 'state', array('NY', 'CA', 'IL', 'TX', 'AZ')[cast(id % 5 as int)]," " 'zip', lpad(cast(id % 99999 as string), 5, '0')," " 'country', 'USA'," " 'geo', named_struct(" " 'lat', cast(id % 180 as double) - 90.0," " 'lng', cast(id % 360 as double) - 180.0," " 'accuracy', cast(id % 100 as double)" " )" ") as address", # Level 3 nested struct: organization hierarchy (70) "named_struct(" " 'company', named_struct(" " 'name', concat('Company_', cast(id % 1000 as string))," " 'industry', array('tech', 'finance', 'healthcare', 'retail')[cast(id % 4 as int)]," " 'headquarters', named_struct(" " 'city', array('NYC', 'SF', 'LA', 'CHI')[cast(id % 4 as int)]," " 'country', 'USA'," " 'timezone', array('EST', 'PST', 'PST', 'CST')[cast(id % 4 as int)]" " )" " )," " 'department', named_struct(" " 'name', array('Engineering', 'Sales', 'Marketing', 'HR')[cast(id % 4 as int)]," " 'code', concat('DEPT-', cast(id % 100 as string))," " 'budget', cast(id % 1000000 as double)" " )" ") as organization", # Level 4 nested struct: deep config (71) "named_struct(" " 'level1', named_struct(" " 'level2a', named_struct(" " 'level3a', named_struct(" " 'value_int', cast(id % 1000 as int)," " 'value_str', concat('deep_', cast(id % 100 as string))," " 'value_bool', id % 2 = 0" " )," " 'level3b', named_struct(" " 'metric1', cast(id % 100 as double)," " 'metric2', cast((id * 2) % 100 as double)" " )" " )," " 'level2b', named_struct(" " 'setting1', concat('setting_', cast(id % 50 as string))," " 'setting2', id % 3 = 0," " 'values', array(cast(id % 10 as int), cast((id + 1) % 10 as int), cast((id + 2) % 10 as int))" " )" " )," " 'metadata', named_struct(" " 'version', concat('v', cast(id % 10 as string))," " 'timestamp', id * 1000" " )" ") as deep_config", # Array of structs with nested structs (72) "array(" " named_struct(" " 'item_id', cast(id % 1000 as int)," " 'details', named_struct(" " 'name', concat('Item_', cast(id % 100 as string))," " 'category', array('electronics', 'clothing', 'food', 'books')[cast(id % 4 as int)]," " 'pricing', named_struct(" " 'base', cast(id % 100 as double) + 0.99," " 'discount', cast(id % 20 as double) / 100.0," " 'tax_rate', 0.08" " )" " )," " 'quantity', cast(id % 10 + 1 as int)" " )," " named_struct(" " 'item_id', cast((id + 100) % 1000 as int)," " 'details', named_struct(" " 'name', concat('Item_', cast((id + 100) % 100 as string))," " 'category', array('electronics', 'clothing', 'food', 'books')[cast((id + 1) % 4 as int)]," " 'pricing', named_struct(" " 'base', cast((id + 50) % 100 as double) + 0.99," " 'discount', cast((id + 5) % 20 as double) / 100.0," " 'tax_rate', 0.08" " )" " )," " 'quantity', cast((id + 1) % 10 + 1 as int)" " )" ") as line_items", # Map with struct values (73) "map(" " 'primary', named_struct('name', concat('Primary_', cast(id % 100 as string)), 'score', cast(id % 100 as double), 'active', true)," " 'secondary', named_struct('name', concat('Secondary_', cast(id % 100 as string)), 'score', cast((id * 2) % 100 as double), 'active', id % 2 = 0)" ") as contact_map", # Struct with map containing arrays (74) "named_struct(" " 'config_name', concat('Config_', cast(id % 100 as string))," " 'settings', map(" " 'integers', array(cast(id % 10 as int), cast((id + 1) % 10 as int), cast((id + 2) % 10 as int))," " 'strings', array(concat('s1_', cast(id % 10 as string)), concat('s2_', cast(id % 10 as string)))" " )," " 'enabled', id % 2 = 0" ") as config_with_map", # Array of arrays (75) "array(" " array(cast(id % 10 as int), cast((id + 1) % 10 as int), cast((id + 2) % 10 as int))," " array(cast((id * 2) % 10 as int), cast((id * 2 + 1) % 10 as int))," " array(cast((id * 3) % 10 as int), cast((id * 3 + 1) % 10 as int), cast((id * 3 + 2) % 10 as int), cast((id * 3 + 3) % 10 as int))" ") as nested_int_arrays", # Array of maps (76) "array(" " map('a', cast(id % 100 as string), 'b', cast((id + 1) % 100 as string))," " map('x', cast((id * 2) % 100 as string), 'y', cast((id * 2 + 1) % 100 as string), 'z', cast((id * 2 + 2) % 100 as string))" ") as array_of_maps", # Map with array values (77) "map(" " 'scores', array(cast(id % 100 as double), cast((id * 2) % 100 as double), cast((id * 3) % 100 as double))," " 'ranks', array(cast(id % 10 as double), cast((id + 1) % 10 as double))" ") as map_with_arrays", # Complex event structure (78) "named_struct(" " 'event_id', concat('EVT-', cast(id as string))," " 'event_type', array('click', 'view', 'purchase', 'signup')[cast(id % 4 as int)]," " 'timestamp', id * 1000," " 'properties', map(" " 'source', array('web', 'mobile', 'api')[cast(id % 3 as int)]," " 'campaign', concat('camp_', cast(id % 50 as string))" " )," " 'user', named_struct(" " 'id', cast(id % 100000 as int)," " 'segment', array('new', 'returning', 'premium')[cast(id % 3 as int)]," " 'attributes', named_struct(" " 'age_group', array('18-24', '25-34', '35-44', '45+')[cast(id % 4 as int)]," " 'interests', array(concat('int_', cast(id % 10 as string)), concat('int_', cast((id + 1) % 10 as string)))" " )" " )" ") as event_data", # Financial transaction with deep nesting (79) "named_struct(" " 'txn_id', concat('TXN-', cast(id as string))," " 'amount', named_struct(" " 'value', cast(id % 10000 as double) / 100.0," " 'currency', array('USD', 'EUR', 'GBP')[cast(id % 3 as int)]," " 'exchange', named_struct(" " 'rate', 1.0 + cast(id % 100 as double) / 1000.0," " 'source', 'market'," " 'timestamp', id * 1000" " )" " )," " 'parties', named_struct(" " 'sender', named_struct(" " 'account', concat('ACC-', lpad(cast(id % 100000 as string), 8, '0'))," " 'bank', named_struct(" " 'code', concat('BNK-', cast(id % 100 as string))," " 'country', array('US', 'UK', 'DE', 'JP')[cast(id % 4 as int)]" " )" " )," " 'receiver', named_struct(" " 'account', concat('ACC-', lpad(cast((id + 50000) % 100000 as string), 8, '0'))," " 'bank', named_struct(" " 'code', concat('BNK-', cast((id + 50) % 100 as string))," " 'country', array('US', 'UK', 'DE', 'JP')[cast((id + 1) % 4 as int)]" " )" " )" " )" ") as financial_txn", # Product catalog entry (80) "named_struct(" " 'product_id', concat('PROD-', lpad(cast(id % 10000 as string), 6, '0'))," " 'variants', array(" " named_struct(" " 'sku', concat('VAR-', cast(id % 1000 as string), '-A')," " 'attributes', map('color', 'red', 'size', 'S')," " 'inventory', named_struct('quantity', cast(id % 100 as int), 'warehouse', 'WH-1')" " )," " named_struct(" " 'sku', concat('VAR-', cast(id % 1000 as string), '-B')," " 'attributes', map('color', 'blue', 'size', 'M')," " 'inventory', named_struct('quantity', cast((id + 10) % 100 as int), 'warehouse', 'WH-2')" " )" " )," " 'pricing', named_struct(" " 'list_price', cast(id % 1000 as double) + 0.99," " 'tiers', array(" " named_struct('min_qty', 1, 'price', cast(id % 1000 as double) + 0.99)," " named_struct('min_qty', 10, 'price', cast(id % 1000 as double) * 0.9 + 0.99)," " named_struct('min_qty', 100, 'price', cast(id % 1000 as double) * 0.8 + 0.99)" " )" " )" ") as product_catalog", # Additional scalar columns (81-90) "cast((id * 53) % 10000 as int) as metric_1", "cast((id * 59) % 10000 as int) as metric_2", "cast((id * 61) % 10000 as int) as metric_3", "cast((id * 67) % 1000000 as long) as counter_1", "cast((id * 71) % 1000000 as long) as counter_2", "cast((id * 73) % 10000 as double) / 100.0 as measure_1", "cast((id * 79) % 10000 as double) / 100.0 as measure_2", "concat('label_', cast(id % 500 as string)) as label_1", "concat('category_', cast(id % 200 as string)) as label_2", "id % 17 = 0 as flag_1", # Additional complex columns (91-95) "array(" " named_struct('ts', id * 1000, 'value', cast(id % 100 as double))," " named_struct('ts', id * 1000 + 1000, 'value', cast((id + 1) % 100 as double))," " named_struct('ts', id * 1000 + 2000, 'value', cast((id + 2) % 100 as double))" ") as time_series", "map(" " 'en', concat('English text ', cast(id % 100 as string))," " 'es', concat('Spanish texto ', cast(id % 100 as string))," " 'fr', concat('French texte ', cast(id % 100 as string))" ") as translations", "named_struct(" " 'rules', array(" " named_struct('id', cast(id % 100 as int), 'condition', concat('cond_', cast(id % 10 as string)), 'action', concat('act_', cast(id % 5 as string)))," " named_struct('id', cast((id + 1) % 100 as int), 'condition', concat('cond_', cast((id + 1) % 10 as string)), 'action', concat('act_', cast((id + 1) % 5 as string)))" " )," " 'default_action', 'none'," " 'priority', cast(id % 10 as int)" ") as rule_engine", "array(" " map('metric', 'cpu', 'value', cast(id % 100 as double), 'unit', 'percent')," " map('metric', 'memory', 'value', cast((id * 2) % 100 as double), 'unit', 'percent')," " map('metric', 'disk', 'value', cast((id * 3) % 100 as double), 'unit', 'percent')" ") as system_metrics", "named_struct(" " 'permissions', map(" " 'read', array('user', 'admin')," " 'write', array('admin')," " 'delete', array('admin')" " )," " 'roles', array(" " named_struct('name', 'viewer', 'level', 1)," " named_struct('name', 'editor', 'level', 2)," " named_struct('name', 'admin', 'level', 3)" " )" ") as access_control", # Final columns (96-100) "cast((id * 83) % 1000 as int) as final_metric_1", "cast((id * 89) % 10000 as double) / 100.0 as final_measure_1", "concat('final_', cast(id % 1000 as string)) as final_label", "array(cast(id % 5 as int), cast((id + 1) % 5 as int), cast((id + 2) % 5 as int), cast((id + 3) % 5 as int), cast((id + 4) % 5 as int)) as final_array", "named_struct(" " 'summary', concat('Summary for record ', cast(id as string))," " 'checksum', concat(cast(id % 256 as string), '-', cast((id * 7) % 256 as string), '-', cast((id * 13) % 256 as string))," " 'version', cast(id % 100 as int)" ") as final_metadata" ) print(f"Generated schema with {len(df.columns)} columns") df.printSchema() # Write as parquet df.write.mode("overwrite").parquet(output_path) # Verify the data written_df = spark.read.parquet(output_path) actual_count = written_df.count() print(f"Wrote {actual_count:,} rows to {output_path}") spark.stop() def main(): parser = argparse.ArgumentParser( description="Generate test data for shuffle benchmark" ) parser.add_argument( "--output", "-o", required=True, help="Output path for parquet data (local path or hdfs://...)" ) parser.add_argument( "--rows", "-r", type=int, default=10_000_000, help="Number of rows to generate (default: 10000000 for ~1GB with wide schema)" ) parser.add_argument( "--partitions", "-p", type=int, default=None, help="Number of output partitions (default: auto based on cluster)" ) args = parser.parse_args() # Default partitions to a reasonable number if not specified num_partitions = args.partitions if args.partitions else 200 generate_data(args.output, args.rows, num_partitions) if __name__ == "__main__": main() ================================================ FILE: benchmarks/pyspark/run_all_benchmarks.sh ================================================ #!/bin/bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # # Run all shuffle benchmarks (Spark, Comet JVM, Comet Native) # Check the Spark UI during each run to compare shuffle sizes set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DATA_PATH="${1:-/tmp/shuffle-benchmark-data}" COMET_JAR="${COMET_JAR:-$SCRIPT_DIR/../../spark/target/comet-spark-spark3.5_2.12-0.14.0-SNAPSHOT.jar}" SPARK_MASTER="${SPARK_MASTER:-local[*]}" EXECUTOR_MEMORY="${EXECUTOR_MEMORY:-16g}" EVENT_LOG_DIR="${EVENT_LOG_DIR:-/tmp/spark-events}" # Create event log directory mkdir -p "$EVENT_LOG_DIR" echo "========================================" echo "Shuffle Size Comparison Benchmark" echo "========================================" echo "Data path: $DATA_PATH" echo "Comet JAR: $COMET_JAR" echo "Spark master: $SPARK_MASTER" echo "Executor memory: $EXECUTOR_MEMORY" echo "Event log dir: $EVENT_LOG_DIR" echo "========================================" # Run Spark baseline (no Comet) echo "" echo ">>> Running SPARK shuffle benchmark..." $SPARK_HOME/bin/spark-submit \ --master "$SPARK_MASTER" \ --executor-memory "$EXECUTOR_MEMORY" \ --conf spark.eventLog.enabled=true \ --conf spark.eventLog.dir="$EVENT_LOG_DIR" \ --conf spark.comet.enabled=false \ --conf spark.comet.exec.shuffle.enabled=false \ "$SCRIPT_DIR/run_benchmark.py" \ --data "$DATA_PATH" \ --mode spark # Run Comet JVM shuffle echo "" echo ">>> Running COMET JVM shuffle benchmark..." $SPARK_HOME/bin/spark-submit \ --master "$SPARK_MASTER" \ --executor-memory "$EXECUTOR_MEMORY" \ --jars "$COMET_JAR" \ --driver-class-path "$COMET_JAR" \ --conf spark.executor.extraClassPath="$COMET_JAR" \ --conf spark.eventLog.enabled=true \ --conf spark.eventLog.dir="$EVENT_LOG_DIR" \ --conf spark.memory.offHeap.enabled=true \ --conf spark.memory.offHeap.size=16g \ --conf spark.comet.enabled=true \ --conf spark.comet.operator.DataWritingCommandExec.allowIncompatible=true \ --conf spark.comet.parquet.write.enabled=true \ --conf spark.comet.logFallbackReasons.enabled=true \ --conf spark.comet.explainFallback.enabled=true \ --conf spark.comet.shuffle.mode=jvm \ --conf spark.comet.exec.shuffle.mode=jvm \ --conf spark.comet.exec.replaceSortMergeJoin=true \ --conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \ --conf spark.sql.extensions=org.apache.comet.CometSparkSessionExtensions \ --conf spark.comet.cast.allowIncompatible=true \ "$SCRIPT_DIR/run_benchmark.py" \ --data "$DATA_PATH" \ --mode jvm # Run Comet Native shuffle echo "" echo ">>> Running COMET NATIVE shuffle benchmark..." $SPARK_HOME/bin/spark-submit \ --master "$SPARK_MASTER" \ --executor-memory "$EXECUTOR_MEMORY" \ --jars "$COMET_JAR" \ --driver-class-path "$COMET_JAR" \ --conf spark.executor.extraClassPath="$COMET_JAR" \ --conf spark.eventLog.enabled=true \ --conf spark.eventLog.dir="$EVENT_LOG_DIR" \ --conf spark.memory.offHeap.enabled=true \ --conf spark.memory.offHeap.size=16g \ --conf spark.comet.enabled=true \ --conf spark.comet.operator.DataWritingCommandExec.allowIncompatible=true \ --conf spark.comet.parquet.write.enabled=true \ --conf spark.comet.logFallbackReasons.enabled=true \ --conf spark.comet.explainFallback.enabled=true \ --conf spark.comet.exec.shuffle.mode=native \ --conf spark.comet.exec.replaceSortMergeJoin=true \ --conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \ --conf spark.sql.extensions=org.apache.comet.CometSparkSessionExtensions \ --conf spark.comet.cast.allowIncompatible=true \ "$SCRIPT_DIR/run_benchmark.py" \ --data "$DATA_PATH" \ --mode native echo "" echo "========================================" echo "BENCHMARK COMPLETE" echo "========================================" echo "Event logs written to: $EVENT_LOG_DIR" echo "" ================================================ FILE: benchmarks/pyspark/run_benchmark.py ================================================ #!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. """ Run PySpark benchmarks. Run benchmarks by name with appropriate spark-submit configs for different modes (spark, jvm, native). Check the Spark UI to compare results between modes. """ import argparse import sys from pyspark.sql import SparkSession from benchmarks import get_benchmark, list_benchmarks def main(): parser = argparse.ArgumentParser( description="Run PySpark benchmarks", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" Examples: # Run hash partitioning shuffle benchmark in Spark mode python run_benchmark.py --data /path/to/data --mode spark --benchmark shuffle-hash # Run round-robin shuffle benchmark in Comet native mode python run_benchmark.py --data /path/to/data --mode native --benchmark shuffle-roundrobin # List all available benchmarks python run_benchmark.py --list-benchmarks """ ) parser.add_argument( "--data", "-d", help="Path to input parquet data" ) parser.add_argument( "--mode", "-m", choices=["spark", "jvm", "native"], help="Shuffle mode being tested" ) parser.add_argument( "--benchmark", "-b", default="shuffle-hash", help="Benchmark to run (default: shuffle-hash)" ) parser.add_argument( "--list-benchmarks", action="store_true", help="List all available benchmarks and exit" ) args = parser.parse_args() # Handle --list-benchmarks if args.list_benchmarks: print("Available benchmarks:\n") for name, description in list_benchmarks(): print(f" {name:25s} - {description}") return 0 # Validate required arguments if not args.data: parser.error("--data is required when running a benchmark") if not args.mode: parser.error("--mode is required when running a benchmark") # Get the benchmark class try: benchmark_cls = get_benchmark(args.benchmark) except KeyError as e: print(f"Error: {e}", file=sys.stderr) print("\nUse --list-benchmarks to see available benchmarks", file=sys.stderr) return 1 # Create Spark session spark = SparkSession.builder \ .appName(f"{benchmark_cls.name()}-{args.mode.upper()}") \ .getOrCreate() try: # Create and run the benchmark benchmark = benchmark_cls(spark, args.data, args.mode) results = benchmark.execute_timed() print("\nCheck Spark UI for shuffle sizes and detailed metrics") return 0 finally: spark.stop() if __name__ == "__main__": sys.exit(main()) ================================================ FILE: benchmarks/tpc/.gitignore ================================================ *.json *.png ================================================ FILE: benchmarks/tpc/README.md ================================================ # Comet Benchmarking Scripts This directory contains scripts used for generating benchmark results that are published in this repository and in the Comet documentation. For full instructions on running these benchmarks on an EC2 instance, see the [Comet Benchmarking on EC2 Guide]. [Comet Benchmarking on EC2 Guide]: https://datafusion.apache.org/comet/contributor-guide/benchmarking_aws_ec2.html ## Setup TPC queries are bundled in `benchmarks/tpc/queries/` (derived from TPC-H/DS under the TPC Fair Use Policy). ## Usage All benchmarks are run via `run.py`: ``` python3 run.py --engine --benchmark [options] ``` | Option | Description | | ------------------------- | ------------------------------------------------------------------------------- | | `--engine` | Engine name (matches a TOML file in `engines/`) | | `--benchmark` | `tpch` or `tpcds` | | `--iterations` | Number of iterations (default: 1) | | `--output` | Output directory (default: `.`) | | `--query` | Run a single query number | | `--no-restart` | Skip Spark master/worker restart | | `--dry-run` | Print the spark-submit command without executing | | `--jfr` | Enable Java Flight Recorder profiling | | `--jfr-dir` | Directory for JFR output files (default: `/results/jfr`) | | `--async-profiler` | Enable async-profiler (profiles Java + native code) | | `--async-profiler-dir` | Directory for async-profiler output (default: `/results/async-profiler`) | | `--async-profiler-event` | Event type: `cpu`, `wall`, `alloc`, `lock`, etc. (default: `cpu`) | | `--async-profiler-format` | Output format: `flamegraph`, `jfr`, `collapsed`, `text` (default: `flamegraph`) | Available engines: `spark`, `comet`, `comet-iceberg`, `gluten` ## Example usage Set Spark environment variables: ```shell export SPARK_HOME=/opt/spark-3.5.3-bin-hadoop3/ export SPARK_MASTER=spark://yourhostname:7077 ``` Set path to data (TPC queries are bundled in `benchmarks/tpc/queries/`): ```shell export TPCH_DATA=/mnt/bigdata/tpch/sf100/ ``` Run Spark benchmark: ```shell export JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64 sudo ./drop-caches.sh python3 run.py --engine spark --benchmark tpch ``` Run Comet benchmark: ```shell export JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64 export COMET_JAR=/opt/comet/comet-spark-spark3.5_2.12-0.10.0.jar sudo ./drop-caches.sh python3 run.py --engine comet --benchmark tpch ``` Run Gluten benchmark: ```shell export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 export GLUTEN_JAR=/opt/gluten/gluten-velox-bundle-spark3.5_2.12-linux_amd64-1.4.0.jar sudo ./drop-caches.sh python3 run.py --engine gluten --benchmark tpch ``` Preview a command without running it: ```shell python3 run.py --engine comet --benchmark tpch --dry-run ``` Generating charts: ```shell python3 generate-comparison.py --benchmark tpch --labels "Spark 3.5.3" "Comet 0.9.0" "Gluten 1.4.0" --title "TPC-H @ 100 GB (single executor, 8 cores, local Parquet files)" spark-tpch-1752338506381.json comet-tpch-1752337818039.json gluten-tpch-1752337474344.json ``` ## Engine Configuration Each engine is defined by a TOML file in `engines/`. The config specifies JARs, Spark conf overrides, required environment variables, and optional defaults/exports. See existing files for examples. ## Iceberg Benchmarking Comet includes native Iceberg support via iceberg-rust integration. This enables benchmarking TPC-H queries against Iceberg tables with native scan acceleration. ### Prerequisites Download the Iceberg Spark runtime JAR (required for running the benchmark): ```shell wget https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-3.5_2.12/1.8.1/iceberg-spark-runtime-3.5_2.12-1.8.1.jar export ICEBERG_JAR=/path/to/iceberg-spark-runtime-3.5_2.12-1.8.1.jar ``` Note: Table creation uses `--packages` which auto-downloads the dependency. ### Create Iceberg tables Convert existing Parquet data to Iceberg format using `create-iceberg-tables.py`. The script configures the Iceberg catalog automatically -- no `--conf` flags needed. ```shell export ICEBERG_WAREHOUSE=/mnt/bigdata/iceberg-warehouse mkdir -p $ICEBERG_WAREHOUSE # TPC-H $SPARK_HOME/bin/spark-submit \ --master $SPARK_MASTER \ --packages org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1 \ --conf spark.driver.memory=8G \ --conf spark.executor.instances=2 \ --conf spark.executor.cores=8 \ --conf spark.cores.max=16 \ --conf spark.executor.memory=16g \ create-iceberg-tables.py \ --benchmark tpch \ --parquet-path $TPCH_DATA \ --warehouse $ICEBERG_WAREHOUSE # TPC-DS $SPARK_HOME/bin/spark-submit \ --master $SPARK_MASTER \ --packages org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1 \ --conf spark.driver.memory=8G \ --conf spark.executor.instances=2 \ --conf spark.executor.cores=8 \ --conf spark.cores.max=16 \ --conf spark.executor.memory=16g \ create-iceberg-tables.py \ --benchmark tpcds \ --parquet-path $TPCDS_DATA \ --warehouse $ICEBERG_WAREHOUSE ``` ### Run Iceberg benchmark ```shell export JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64 export COMET_JAR=/opt/comet/comet-spark-spark3.5_2.12-0.10.0.jar export ICEBERG_JAR=/path/to/iceberg-spark-runtime-3.5_2.12-1.8.1.jar export ICEBERG_WAREHOUSE=/mnt/bigdata/iceberg-warehouse sudo ./drop-caches.sh python3 run.py --engine comet-iceberg --benchmark tpch ``` The benchmark uses Comet's native iceberg-rust integration, which is enabled by default. Verify native scanning is active by checking for `CometIcebergNativeScanExec` in the physical plan output. ### create-iceberg-tables.py options | Option | Required | Default | Description | | ---------------- | -------- | -------------- | ----------------------------------- | | `--benchmark` | Yes | | `tpch` or `tpcds` | | `--parquet-path` | Yes | | Path to source Parquet data | | `--warehouse` | Yes | | Path to Iceberg warehouse directory | | `--catalog` | No | `local` | Iceberg catalog name | | `--database` | No | benchmark name | Database name for the tables | ## Running with Docker A Docker Compose setup is provided in `infra/docker/` for running benchmarks in an isolated Spark standalone cluster. The Docker image supports both **Linux (amd64)** and **macOS (arm64)** via architecture-agnostic Java symlinks created at build time. ### Build the image The image must be built for the correct platform to match the native libraries in the engine JARs (e.g. Comet bundles `libcomet.so` for a specific OS/arch). ```shell docker build -t comet-bench -f benchmarks/tpc/infra/docker/Dockerfile . ``` ### Building a compatible Comet JAR The Comet JAR contains platform-specific native libraries (`libcomet.so` / `libcomet.dylib`). A JAR built on the host may not work inside the Docker container due to OS, architecture, or glibc version mismatches. Use `Dockerfile.build-comet` to build a JAR with compatible native libraries: - **macOS (Apple Silicon):** The host JAR contains `darwin/aarch64` libraries which won't work in Linux containers. You **must** use the build Dockerfile. - **Linux:** If your host glibc version differs from the container's, the native library will fail to load with a `GLIBC_x.xx not found` error. The build Dockerfile uses Ubuntu 20.04 (glibc 2.31) for broad compatibility. Use it if you see `UnsatisfiedLinkError` mentioning glibc when running benchmarks. ```shell mkdir -p output docker build -t comet-builder \ -f benchmarks/tpc/infra/docker/Dockerfile.build-comet . docker run --rm -v $(pwd)/output:/output comet-builder export COMET_JAR=$(pwd)/output/comet-spark-spark3.5_2.12-*.jar ``` ### Platform notes **macOS (Apple Silicon):** Docker Desktop is required. - **Memory:** Docker Desktop defaults to a small memory allocation (often 8 GB) which is not enough for Spark benchmarks. Go to **Docker Desktop > Settings > Resources > Memory** and increase it to at least 48 GB (each worker requests 16 GB for its executor plus overhead, and the driver needs 8 GB). Without enough memory, executors will be OOM-killed (exit code 137). - **File Sharing:** You may need to add your data directory (e.g. `/opt`) to **Docker Desktop > Settings > Resources > File Sharing** before mounting host volumes. **Linux (amd64):** Docker uses cgroup memory limits directly without a VM layer. No special Docker configuration is needed, but you may still need to build the Comet JAR using `Dockerfile.build-comet` (see above) if your host glibc version doesn't match the container's. The Docker image auto-detects the container architecture (amd64/arm64) and sets up arch-agnostic Java symlinks. The compose file uses `BENCH_JAVA_HOME` (not `JAVA_HOME`) to avoid inheriting the host's Java path into the container. ### Start the cluster Set environment variables pointing to your host paths, then start the Spark master and two workers: ```shell export DATA_DIR=/mnt/bigdata/tpch/sf100 export RESULTS_DIR=/tmp/bench-results export COMET_JAR=/opt/comet/comet-spark-spark3.5_2.12-0.10.0.jar mkdir -p $RESULTS_DIR/spark-events docker compose -f benchmarks/tpc/infra/docker/docker-compose.yml up -d ``` Set `COMET_JAR`, `GLUTEN_JAR`, or `ICEBERG_JAR` to the host path of the engine JAR you want to use. Each JAR is mounted individually into the container, so you can easily switch between versions by changing the path and restarting. ### Run benchmarks Use `docker compose run --rm` to execute benchmarks. The `--rm` flag removes the container when it exits, preventing port conflicts on subsequent runs. Pass `--no-restart` since the cluster is already managed by Compose, and `--output /results` so that output files land in the mounted results directory: ```shell docker compose -f benchmarks/tpc/infra/docker/docker-compose.yml \ run --rm -p 4040:4040 bench \ python3 /opt/benchmarks/run.py \ --engine comet --benchmark tpch --output /results --no-restart ``` The `-p 4040:4040` flag exposes the Spark Application UI on the host. The following UIs are available during a benchmark run: | UI | URL | | ----------------- | ---------------------- | | Spark Master | http://localhost:8080 | | Worker 1 | http://localhost:8081 | | Worker 2 | http://localhost:8082 | | Spark Application | http://localhost:4040 | | History Server | http://localhost:18080 | > **Note:** The Master UI links to the Application UI using the container's internal > hostname, which is not reachable from the host. Use `http://localhost:4040` directly > to access the Application UI. The Spark Application UI is only available while a benchmark is running. To inspect completed runs, uncomment the `history-server` service in `docker-compose.yml` and restart the cluster. The History Server reads event logs from `$RESULTS_DIR/spark-events`. For Gluten (requires Java 8), you must restart the **entire cluster** with `JAVA_HOME` set so that all services (master, workers, and bench) use Java 8: ```shell export BENCH_JAVA_HOME=/usr/lib/jvm/java-8-openjdk docker compose -f benchmarks/tpc/infra/docker/docker-compose.yml down docker compose -f benchmarks/tpc/infra/docker/docker-compose.yml up -d docker compose -f benchmarks/tpc/infra/docker/docker-compose.yml \ run --rm bench \ python3 /opt/benchmarks/run.py \ --engine gluten --benchmark tpch --output /results --no-restart ``` > **Important:** Only passing `-e JAVA_HOME=...` to the `bench` container is not > sufficient -- the workers also need Java 8 or Gluten will fail at runtime with > `sun.misc.Unsafe` errors. Unset `BENCH_JAVA_HOME` (or switch it back to Java 17) > and restart the cluster before running Comet or Spark benchmarks. ### Memory limits Two compose files are provided for different hardware profiles: | File | Workers | Total memory | Use case | | --------------------------- | ------- | ------------ | ------------------------------ | | `docker-compose.yml` | 2 | ~74 GB | SF100+ on a workstation/server | | `docker-compose-laptop.yml` | 1 | ~12 GB | SF1–SF10 on a laptop | **`docker-compose.yml`** (workstation default): | Container | Container limit (`mem_limit`) | Spark JVM allocation | | -------------- | ----------------------------- | ------------------------- | | spark-worker-1 | 32 GB | 16 GB executor + overhead | | spark-worker-2 | 32 GB | 16 GB executor + overhead | | bench (driver) | 10 GB | 8 GB driver | | **Total** | **74 GB** | | Configure via environment variables: `WORKER_MEM_LIMIT` (default: 32g per worker), `BENCH_MEM_LIMIT` (default: 10g), `WORKER_MEMORY` (default: 16g, Spark executor memory), `WORKER_CORES` (default: 8). ### Running on a laptop with small scale factors For local development or testing with small scale factors (e.g. SF1 or SF10), use the laptop compose file which runs a single worker with reduced memory: ```shell docker compose -f benchmarks/tpc/infra/docker/docker-compose-laptop.yml up -d ``` This starts one worker (4 GB executor inside an 8 GB container) and a 4 GB bench container, totaling approximately **12 GB** of memory. The benchmark scripts request 2 executor instances and 16 max cores by default (`run.py`). Spark will simply use whatever resources are available on the single worker, so no script changes are needed. ### Comparing Parquet vs Iceberg performance Run both benchmarks and compare: ```shell python3 generate-comparison.py --benchmark tpch \ --labels "Comet (Parquet)" "Comet (Iceberg)" \ --title "TPC-H @ 100 GB: Parquet vs Iceberg" \ comet-tpch-*.json comet-iceberg-tpch-*.json ``` ## Java Flight Recorder Profiling Use the `--jfr` flag to capture JFR profiles from the Spark driver and executors. JFR is built into JDK 11+ so no additional dependencies are needed. ```shell python3 run.py --engine comet --benchmark tpch --jfr ``` JFR recordings are written to `/results/jfr/` by default (configurable with `--jfr-dir`). The driver writes `driver.jfr` and each executor writes `executor.jfr` (JFR appends the PID when multiple executors share a path). With Docker Compose, the `/results` volume is shared across all containers, so JFR files from both driver and executors are collected in `$RESULTS_DIR/jfr/` on the host: ```shell docker compose -f benchmarks/tpc/infra/docker/docker-compose.yml \ run --rm bench \ python3 /opt/benchmarks/run.py \ --engine comet --benchmark tpch --output /results --no-restart --jfr ``` Open the `.jfr` files with [JDK Mission Control](https://jdk.java.net/jmc/), IntelliJ IDEA's profiler, or `jfr` CLI tool (`jfr summary driver.jfr`). ## async-profiler Profiling Use the `--async-profiler` flag to capture profiles with [async-profiler](https://github.com/async-profiler/async-profiler). Unlike JFR, async-profiler can profile **both Java and native (Rust/C++) code** in the same flame graph, making it especially useful for profiling Comet workloads. ### Prerequisites async-profiler must be installed on every node where the driver or executors run. Set `ASYNC_PROFILER_HOME` to the installation directory: ```shell # Download and extract (Linux x64 example) wget https://github.com/async-profiler/async-profiler/releases/download/v3.0/async-profiler-3.0-linux-x64.tar.gz tar xzf async-profiler-3.0-linux-x64.tar.gz -C /opt/async-profiler --strip-components=1 export ASYNC_PROFILER_HOME=/opt/async-profiler ``` On Linux, `perf_event_paranoid` must be set to allow profiling: ```shell sudo sysctl kernel.perf_event_paranoid=1 # or 0 / -1 for full access sudo sysctl kernel.kptr_restrict=0 # optional: enable kernel symbols ``` ### Basic usage ```shell python3 run.py --engine comet --benchmark tpch --async-profiler ``` This produces HTML flame graphs in `/results/async-profiler/` by default (`driver.html` and `executor.html`). ### Choosing events and output format ```shell # Wall-clock profiling (includes time spent waiting/sleeping) python3 run.py --engine comet --benchmark tpch \ --async-profiler --async-profiler-event wall # Allocation profiling with JFR output python3 run.py --engine comet --benchmark tpch \ --async-profiler --async-profiler-event alloc --async-profiler-format jfr # Lock contention profiling python3 run.py --engine comet --benchmark tpch \ --async-profiler --async-profiler-event lock ``` | Event | Description | | ------- | --------------------------------------------------- | | `cpu` | On-CPU time (default). Shows where CPU cycles go. | | `wall` | Wall-clock time. Includes threads that are blocked. | | `alloc` | Heap allocation profiling. | | `lock` | Lock contention profiling. | | Format | Extension | Description | | ------------ | --------- | ---------------------------------------- | | `flamegraph` | `.html` | Interactive HTML flame graph (default). | | `jfr` | `.jfr` | JFR format, viewable in JMC or IntelliJ. | | `collapsed` | `.txt` | Collapsed stacks for FlameGraph scripts. | | `text` | `.txt` | Flat text summary of hot methods. | ### Docker usage The Docker image includes async-profiler pre-installed at `/opt/async-profiler`. The `ASYNC_PROFILER_HOME` environment variable is already set in the compose files, so no extra configuration is needed: ```shell docker compose -f benchmarks/tpc/infra/docker/docker-compose.yml \ run --rm bench \ python3 /opt/benchmarks/run.py \ --engine comet --benchmark tpch --output /results --no-restart --async-profiler ``` Output files are collected in `$RESULTS_DIR/async-profiler/` on the host. **Note:** On Linux, the Docker container needs `--privileged` or `SYS_PTRACE` capability and `perf_event_paranoid <= 1` on the host for `cpu`/`wall` events. Allocation (`alloc`) and lock (`lock`) events work without special privileges. ================================================ FILE: benchmarks/tpc/create-iceberg-tables.py ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. """ Convert TPC-H or TPC-DS Parquet data to Iceberg tables. Usage: spark-submit \ --packages org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1 \ create-iceberg-tables.py \ --benchmark tpch \ --parquet-path /path/to/tpch/parquet \ --warehouse /path/to/iceberg-warehouse spark-submit \ --packages org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1 \ create-iceberg-tables.py \ --benchmark tpcds \ --parquet-path /path/to/tpcds/parquet \ --warehouse /path/to/iceberg-warehouse """ import argparse import os import sys from pyspark.sql import SparkSession import time TPCH_TABLES = [ "customer", "lineitem", "nation", "orders", "part", "partsupp", "region", "supplier", ] TPCDS_TABLES = [ "call_center", "catalog_page", "catalog_returns", "catalog_sales", "customer", "customer_address", "customer_demographics", "date_dim", "time_dim", "household_demographics", "income_band", "inventory", "item", "promotion", "reason", "ship_mode", "store", "store_returns", "store_sales", "warehouse", "web_page", "web_returns", "web_sales", "web_site", ] BENCHMARK_TABLES = { "tpch": TPCH_TABLES, "tpcds": TPCDS_TABLES, } def main(benchmark: str, parquet_path: str, warehouse: str, catalog: str, database: str): table_names = BENCHMARK_TABLES[benchmark] # Validate paths before starting Spark errors = [] if not os.path.isdir(parquet_path): errors.append(f"Error: --parquet-path '{parquet_path}' does not exist or is not a directory") if not os.path.isdir(warehouse): errors.append(f"Error: --warehouse '{warehouse}' does not exist or is not a directory. " "Create it with: mkdir -p " + warehouse) if errors: for e in errors: print(e, file=sys.stderr) sys.exit(1) spark = SparkSession.builder \ .appName(f"Create Iceberg {benchmark.upper()} Tables") \ .config(f"spark.sql.catalog.{catalog}", "org.apache.iceberg.spark.SparkCatalog") \ .config(f"spark.sql.catalog.{catalog}.type", "hadoop") \ .config(f"spark.sql.catalog.{catalog}.warehouse", warehouse) \ .getOrCreate() # Set the Iceberg catalog as the current catalog so that # namespace operations are routed correctly spark.sql(f"USE {catalog}") # Create namespace if it doesn't exist try: spark.sql(f"CREATE NAMESPACE IF NOT EXISTS {database}") except Exception: # Namespace may already exist pass for table in table_names: parquet_table_path = f"{parquet_path}/{table}.parquet" iceberg_table = f"{catalog}.{database}.{table}" print(f"Converting {parquet_table_path} -> {iceberg_table}") start_time = time.time() # Drop table if exists to allow re-running spark.sql(f"DROP TABLE IF EXISTS {iceberg_table}") # Read parquet and write as Iceberg df = spark.read.parquet(parquet_table_path) df.writeTo(iceberg_table).using("iceberg").create() row_count = spark.table(iceberg_table).count() elapsed = time.time() - start_time print(f" Created {iceberg_table} with {row_count} rows in {elapsed:.2f}s") print(f"\nAll {benchmark.upper()} tables created successfully!") print(f"Tables available at: {catalog}.{database}.*") spark.stop() if __name__ == "__main__": parser = argparse.ArgumentParser( description="Convert TPC-H or TPC-DS Parquet data to Iceberg tables" ) parser.add_argument( "--benchmark", required=True, choices=["tpch", "tpcds"], help="Benchmark whose tables to convert (tpch or tpcds)" ) parser.add_argument( "--parquet-path", required=True, help="Path to Parquet data directory" ) parser.add_argument( "--warehouse", required=True, help="Path to Iceberg warehouse directory" ) parser.add_argument( "--catalog", default="local", help="Iceberg catalog name (default: 'local')" ) parser.add_argument( "--database", default=None, help="Database name to create tables in (defaults to benchmark name)" ) args = parser.parse_args() database = args.database if args.database else args.benchmark main(args.benchmark, args.parquet_path, args.warehouse, args.catalog, database) ================================================ FILE: benchmarks/tpc/drop-caches.sh ================================================ #!/bin/bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # echo 1 > /proc/sys/vm/drop_caches ================================================ FILE: benchmarks/tpc/engines/comet-hashjoin.toml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. [engine] name = "comet-hashjoin" [env] required = ["COMET_JAR"] [spark_submit] jars = ["$COMET_JAR"] driver_class_path = ["$COMET_JAR"] [spark_conf] "spark.driver.extraClassPath" = "$COMET_JAR" "spark.executor.extraClassPath" = "$COMET_JAR" "spark.plugins" = "org.apache.spark.CometPlugin" "spark.shuffle.manager" = "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager" "spark.comet.scan.impl" = "native_datafusion" "spark.comet.exec.replaceSortMergeJoin" = "true" "spark.comet.expression.Cast.allowIncompatible" = "true" ================================================ FILE: benchmarks/tpc/engines/comet-iceberg-hashjoin.toml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. [engine] name = "comet-iceberg-hashjoin" [env] required = ["COMET_JAR", "ICEBERG_JAR", "ICEBERG_WAREHOUSE"] [env.defaults] ICEBERG_CATALOG = "local" [spark_submit] jars = ["$COMET_JAR", "$ICEBERG_JAR"] driver_class_path = ["$COMET_JAR", "$ICEBERG_JAR"] [spark_conf] "spark.driver.extraClassPath" = "$COMET_JAR:$ICEBERG_JAR" "spark.executor.extraClassPath" = "$COMET_JAR:$ICEBERG_JAR" "spark.plugins" = "org.apache.spark.CometPlugin" "spark.shuffle.manager" = "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager" "spark.comet.exec.replaceSortMergeJoin" = "true" "spark.comet.expression.Cast.allowIncompatible" = "true" "spark.comet.enabled" = "true" "spark.comet.exec.enabled" = "true" "spark.comet.scan.icebergNative.enabled" = "true" "spark.comet.explainFallback.enabled" = "true" "spark.sql.catalog.${ICEBERG_CATALOG}" = "org.apache.iceberg.spark.SparkCatalog" "spark.sql.catalog.${ICEBERG_CATALOG}.type" = "hadoop" "spark.sql.catalog.${ICEBERG_CATALOG}.warehouse" = "$ICEBERG_WAREHOUSE" "spark.sql.defaultCatalog" = "${ICEBERG_CATALOG}" [tpcbench_args] use_iceberg = true ================================================ FILE: benchmarks/tpc/engines/comet-iceberg.toml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. [engine] name = "comet-iceberg" [env] required = ["COMET_JAR", "ICEBERG_JAR", "ICEBERG_WAREHOUSE"] [env.defaults] ICEBERG_CATALOG = "local" [spark_submit] jars = ["$COMET_JAR", "$ICEBERG_JAR"] driver_class_path = ["$COMET_JAR", "$ICEBERG_JAR"] [spark_conf] "spark.driver.extraClassPath" = "$COMET_JAR:$ICEBERG_JAR" "spark.executor.extraClassPath" = "$COMET_JAR:$ICEBERG_JAR" "spark.plugins" = "org.apache.spark.CometPlugin" "spark.shuffle.manager" = "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager" "spark.comet.expression.Cast.allowIncompatible" = "true" "spark.comet.enabled" = "true" "spark.comet.exec.enabled" = "true" "spark.comet.scan.icebergNative.enabled" = "true" "spark.comet.explainFallback.enabled" = "true" "spark.sql.catalog.${ICEBERG_CATALOG}" = "org.apache.iceberg.spark.SparkCatalog" "spark.sql.catalog.${ICEBERG_CATALOG}.type" = "hadoop" "spark.sql.catalog.${ICEBERG_CATALOG}.warehouse" = "$ICEBERG_WAREHOUSE" "spark.sql.defaultCatalog" = "${ICEBERG_CATALOG}" [tpcbench_args] use_iceberg = true ================================================ FILE: benchmarks/tpc/engines/comet.toml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. [engine] name = "comet" [env] required = ["COMET_JAR"] [spark_submit] jars = ["$COMET_JAR"] driver_class_path = ["$COMET_JAR"] [spark_conf] "spark.driver.extraClassPath" = "$COMET_JAR" "spark.executor.extraClassPath" = "$COMET_JAR" "spark.plugins" = "org.apache.spark.CometPlugin" "spark.shuffle.manager" = "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager" "spark.comet.scan.impl" = "native_datafusion" "spark.comet.expression.Cast.allowIncompatible" = "true" ================================================ FILE: benchmarks/tpc/engines/gluten.toml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. [engine] name = "gluten" [env] required = ["GLUTEN_JAR"] exports = { TZ = "UTC" } [spark_submit] jars = ["$GLUTEN_JAR"] [spark_conf] "spark.plugins" = "org.apache.gluten.GlutenPlugin" "spark.driver.extraClassPath" = "${GLUTEN_JAR}" "spark.executor.extraClassPath" = "${GLUTEN_JAR}" "spark.gluten.sql.columnar.forceShuffledHashJoin" = "true" "spark.shuffle.manager" = "org.apache.spark.shuffle.sort.ColumnarShuffleManager" "spark.sql.session.timeZone" = "UTC" ================================================ FILE: benchmarks/tpc/engines/spark.toml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. [engine] name = "spark" ================================================ FILE: benchmarks/tpc/generate-comparison.py ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. import argparse import json import logging import matplotlib.pyplot as plt import numpy as np logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) def geomean(data): return np.prod(data) ** (1 / len(data)) def get_durations(result, query_key): """Extract durations from a query result, supporting both old (list) and new (dict) formats.""" value = result[query_key] if isinstance(value, dict): return value["durations"] return value def get_all_queries(results): """Return the sorted union of all query keys across all result sets.""" all_keys = set() for result in results: all_keys.update(result.keys()) # Filter to numeric query keys and sort numerically numeric_keys = [] for k in all_keys: try: numeric_keys.append(int(k)) except ValueError: pass return sorted(numeric_keys) def get_common_queries(results, labels): """Return queries present in ALL result sets, warning about queries missing from some files.""" all_queries = get_all_queries(results) common = [] for query in all_queries: key = str(query) present = [labels[i] for i, r in enumerate(results) if key in r] missing = [labels[i] for i, r in enumerate(results) if key not in r] if missing: logger.warning(f"Query {query}: present in [{', '.join(present)}] but missing from [{', '.join(missing)}]") if not missing: common.append(query) return common def check_result_consistency(results, labels, benchmark): """Log warnings if row counts or result hashes differ across result sets.""" all_queries = get_all_queries(results) for query in all_queries: key = str(query) row_counts = [] hashes = [] for i, result in enumerate(results): if key not in result: continue value = result[key] if not isinstance(value, dict): continue if "row_count" in value: row_counts.append((labels[i], value["row_count"])) if "result_hash" in value: hashes.append((labels[i], value["result_hash"])) if len(row_counts) > 1: counts = set(rc for _, rc in row_counts) if len(counts) > 1: details = ", ".join(f"{label}={rc}" for label, rc in row_counts) logger.warning(f"Query {query}: row count mismatch: {details}") if len(hashes) > 1: hash_values = set(h for _, h in hashes) if len(hash_values) > 1: details = ", ".join(f"{label}={h}" for label, h in hashes) logger.warning(f"Query {query}: result hash mismatch: {details}") def generate_query_rel_speedup_chart(baseline, comparison, label1: str, label2: str, benchmark: str, title: str, common_queries=None): if common_queries is None: common_queries = range(1, query_count(benchmark)+1) results = [] for query in common_queries: a = np.median(np.array(get_durations(baseline, str(query)))) b = np.median(np.array(get_durations(comparison, str(query)))) if a > b: speedup = a/b-1 else: speedup = -(1/(a/b)-1) results.append(("q" + str(query), round(speedup*100, 0))) results = sorted(results, key=lambda x: -x[1]) queries, speedups = zip(*results) # Create figure and axis if benchmark == "tpch": fig, ax = plt.subplots(figsize=(10, 6)) else: fig, ax = plt.subplots(figsize=(35, 10)) # Create bar chart bars = ax.bar(queries, speedups, color='skyblue') # Add text annotations for bar, speedup in zip(bars, speedups): yval = bar.get_height() if yval >= 0: ax.text(bar.get_x() + bar.get_width() / 2.0, min(800, yval+5), f'{yval:.0f}%', va='bottom', ha='center', fontsize=8, color='blue', rotation=90) else: ax.text(bar.get_x() + bar.get_width() / 2.0, yval, f'{yval:.0f}%', va='top', ha='center', fontsize=8, color='blue', rotation=90) # Add title and labels ax.set_title(label2 + " speedup over " + label1 + " (" + title + ")") ax.set_ylabel('Speedup Percentage (100% speedup = 2x faster)') ax.set_xlabel('Query') # Customize the y-axis to handle both positive and negative values better ax.axhline(0, color='black', linewidth=0.8) min_value = (min(speedups) // 100) * 100 max_value = ((max(speedups) // 100) + 1) * 100 + 50 if benchmark == "tpch": ax.set_ylim(min_value, max_value) else: # TODO improve this ax.set_ylim(-250, 300) # Show grid for better readability ax.yaxis.grid(True) # Save the plot as an image file plt.savefig(f'{benchmark}_queries_speedup_rel.png', format='png') def generate_query_abs_speedup_chart(baseline, comparison, label1: str, label2: str, benchmark: str, title: str, common_queries=None): if common_queries is None: common_queries = range(1, query_count(benchmark)+1) results = [] for query in common_queries: a = np.median(np.array(get_durations(baseline, str(query)))) b = np.median(np.array(get_durations(comparison, str(query)))) speedup = a-b results.append(("q" + str(query), round(speedup, 1))) results = sorted(results, key=lambda x: -x[1]) queries, speedups = zip(*results) # Create figure and axis if benchmark == "tpch": fig, ax = plt.subplots(figsize=(10, 6)) else: fig, ax = plt.subplots(figsize=(35, 10)) # Create bar chart bars = ax.bar(queries, speedups, color='skyblue') # Add text annotations for bar, speedup in zip(bars, speedups): yval = bar.get_height() if yval >= 0: ax.text(bar.get_x() + bar.get_width() / 2.0, min(800, yval+5), f'{yval:.1f}', va='bottom', ha='center', fontsize=8, color='blue', rotation=90) else: ax.text(bar.get_x() + bar.get_width() / 2.0, yval, f'{yval:.1f}', va='top', ha='center', fontsize=8, color='blue', rotation=90) # Add title and labels ax.set_title(label2 + " speedup over " + label1 + " (" + title + ")") ax.set_ylabel('Speedup (in seconds)') ax.set_xlabel('Query') # Customize the y-axis to handle both positive and negative values better ax.axhline(0, color='black', linewidth=0.8) min_value = min(speedups) * 2 - 20 max_value = max(speedups) * 1.5 ax.set_ylim(min_value, max_value) # Show grid for better readability ax.yaxis.grid(True) # Save the plot as an image file plt.savefig(f'{benchmark}_queries_speedup_abs.png', format='png') def generate_query_comparison_chart(results, labels, benchmark: str, title: str, common_queries=None): if common_queries is None: common_queries = range(1, query_count(benchmark)+1) queries = [] benches = [] for _ in results: benches.append([]) for query in common_queries: queries.append("q" + str(query)) for i in range(0, len(results)): benches[i].append(np.median(np.array(get_durations(results[i], str(query))))) # Define the width of the bars bar_width = 0.3 # Define the positions of the bars on the x-axis index = np.arange(len(queries)) * 1.5 # Create a bar chart if benchmark == "tpch": fig, ax = plt.subplots(figsize=(15, 6)) else: fig, ax = plt.subplots(figsize=(35, 6)) for i in range(0, len(results)): bar = ax.bar(index + i * bar_width, benches[i], bar_width, label=labels[i]) # Add labels, title, and legend ax.set_title(title) ax.set_xlabel('Queries') ax.set_ylabel('Query Time (seconds)') ax.set_xticks(index + bar_width / 2) ax.set_xticklabels(queries) ax.legend() # Save the plot as an image file plt.savefig(f'{benchmark}_queries_compare.png', format='png') def generate_summary(results, labels, benchmark: str, title: str, common_queries=None): if common_queries is None: common_queries = range(1, query_count(benchmark)+1) timings = [] for _ in results: timings.append(0) num_queries = len([q for q in common_queries]) for query in common_queries: for i in range(0, len(results)): timings[i] += np.median(np.array(get_durations(results[i], str(query)))) # Create figure and axis fig, ax = plt.subplots() fig.set_size_inches(10, 6) # Add title and labels ax.set_title(title) ax.set_ylabel(f'Time in seconds to run {num_queries} {benchmark} queries (lower is better)') times = [round(x,0) for x in timings] # Create bar chart bars = ax.bar(labels, times, color='skyblue', width=0.8) # Add text annotations for bar in bars: yval = bar.get_height() ax.text(bar.get_x() + bar.get_width() / 2.0, yval, f'{yval}', va='bottom') # va: vertical alignment plt.savefig(f'{benchmark}_allqueries.png', format='png') def query_count(benchmark: str): if benchmark == "tpch": return 22 elif benchmark == "tpcds": return 99 else: raise "invalid benchmark name" def main(files, labels, benchmark: str, title: str): results = [] for filename in files: with open(filename) as f: results.append(json.load(f)) check_result_consistency(results, labels, benchmark) common_queries = get_common_queries(results, labels) if not common_queries: logger.error("No queries found in common across all result files") return generate_summary(results, labels, benchmark, title, common_queries) generate_query_comparison_chart(results, labels, benchmark, title, common_queries) if len(files) == 2: generate_query_abs_speedup_chart(results[0], results[1], labels[0], labels[1], benchmark, title, common_queries) generate_query_rel_speedup_chart(results[0], results[1], labels[0], labels[1], benchmark, title, common_queries) if __name__ == '__main__': argparse = argparse.ArgumentParser(description='Generate comparison') argparse.add_argument('filenames', nargs='+', type=str, help='JSON result files') argparse.add_argument('--labels', nargs='+', type=str, help='Labels') argparse.add_argument('--benchmark', type=str, help='Benchmark name (tpch or tpcds)') argparse.add_argument('--title', type=str, help='Chart title') args = argparse.parse_args() main(args.filenames, args.labels, args.benchmark, args.title) ================================================ FILE: benchmarks/tpc/infra/docker/Dockerfile ================================================ # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Benchmark image for running TPC-H and TPC-DS benchmarks across engines # (Spark, Comet, Gluten). # # Build (from repository root): # docker build -t comet-bench -f benchmarks/tpc/infra/docker/Dockerfile . ARG SPARK_IMAGE=apache/spark:3.5.2-python3 FROM ${SPARK_IMAGE} USER root # Install Java 8 (Gluten) and Java 17 (Comet) plus Python 3. RUN apt-get update \ && apt-get install -y --no-install-recommends \ openjdk-8-jdk-headless \ openjdk-17-jdk-headless \ python3 python3-pip procps wget \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* # Install async-profiler for profiling Java + native (Rust/C++) code. ARG ASYNC_PROFILER_VERSION=3.0 RUN ARCH=$(uname -m) && \ if [ "$ARCH" = "x86_64" ]; then AP_ARCH="linux-x64"; \ elif [ "$ARCH" = "aarch64" ]; then AP_ARCH="linux-aarch64"; \ else echo "Unsupported architecture: $ARCH" && exit 1; fi && \ wget -q "https://github.com/async-profiler/async-profiler/releases/download/v${ASYNC_PROFILER_VERSION}/async-profiler-${ASYNC_PROFILER_VERSION}-${AP_ARCH}.tar.gz" \ -O /tmp/async-profiler.tar.gz && \ mkdir -p /opt/async-profiler && \ tar xzf /tmp/async-profiler.tar.gz -C /opt/async-profiler --strip-components=1 && \ rm /tmp/async-profiler.tar.gz ENV ASYNC_PROFILER_HOME=/opt/async-profiler # Default to Java 17 (override with JAVA_HOME at runtime for Gluten). # Detect architecture (amd64 or arm64) so the image works on both Linux and macOS. ARG TARGETARCH RUN ln -s /usr/lib/jvm/java-17-openjdk-${TARGETARCH} /usr/lib/jvm/java-17-openjdk && \ ln -s /usr/lib/jvm/java-8-openjdk-${TARGETARCH} /usr/lib/jvm/java-8-openjdk ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk # Copy the benchmark scripts into the image. COPY benchmarks/tpc/run.py /opt/benchmarks/run.py COPY benchmarks/tpc/tpcbench.py /opt/benchmarks/tpcbench.py COPY benchmarks/tpc/engines /opt/benchmarks/engines COPY benchmarks/tpc/queries /opt/benchmarks/queries COPY benchmarks/tpc/create-iceberg-tables.py /opt/benchmarks/create-iceberg-tables.py COPY benchmarks/tpc/generate-comparison.py /opt/benchmarks/generate-comparison.py # Engine JARs are bind-mounted or copied in at runtime via --jars. # Data and query paths are also bind-mounted. WORKDIR /opt/benchmarks # Defined in the base apache/spark image. ARG spark_uid USER ${spark_uid} ================================================ FILE: benchmarks/tpc/infra/docker/Dockerfile.build-comet ================================================ # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Build a Comet JAR with native libraries for the current platform. # # This is useful on macOS (Apple Silicon) where the host-built JAR contains # darwin/aarch64 native libraries but Docker containers need linux/aarch64. # # Usage (from repository root): # docker build -t comet-builder -f benchmarks/tpc/infra/docker/Dockerfile.build-comet . # docker run --rm -v $(pwd)/output:/output comet-builder # # The JAR is copied to ./output/ on the host. # Use Ubuntu 20.04 to match the GLIBC version (2.31) in apache/spark images. FROM ubuntu:20.04 AS builder ARG TARGETARCH ENV DEBIAN_FRONTEND=noninteractive # Install build dependencies: Java 17, Maven wrapper prerequisites, GCC 11. # Ubuntu 20.04's default GCC 9 has a memcmp bug (GCC #95189) that breaks aws-lc-sys. RUN apt-get update && apt-get install -y --no-install-recommends \ openjdk-17-jdk-headless \ curl ca-certificates git pkg-config \ libssl-dev unzip software-properties-common \ && add-apt-repository -y ppa:ubuntu-toolchain-r/test \ && apt-get update \ && apt-get install -y --no-install-recommends gcc-11 g++-11 make \ && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 110 \ && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 110 \ && update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-11 110 \ && apt-get clean && rm -rf /var/lib/apt/lists/* # Install protoc 25.x (Ubuntu 22.04's protoc is too old for proto3 optional fields). ARG PROTOC_VERSION=25.6 RUN ARCH=$(uname -m) && \ if [ "$ARCH" = "aarch64" ]; then PROTOC_ARCH="linux-aarch_64"; \ else PROTOC_ARCH="linux-x86_64"; fi && \ curl -sLO "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-${PROTOC_ARCH}.zip" && \ unzip -o "protoc-${PROTOC_VERSION}-${PROTOC_ARCH}.zip" -d /usr/local bin/protoc && \ rm "protoc-${PROTOC_VERSION}-${PROTOC_ARCH}.zip" && \ protoc --version # Set JAVA_HOME and LD_LIBRARY_PATH so the Rust build can find libjvm. RUN ln -s /usr/lib/jvm/java-17-openjdk-${TARGETARCH} /usr/lib/jvm/java-17-openjdk ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk ENV LD_LIBRARY_PATH=${JAVA_HOME}/lib/server # Install Rust. RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ENV PATH="/root/.cargo/bin:${PATH}" WORKDIR /build # Copy the full source tree. COPY . . # Build native code + package the JAR (skip tests). RUN make release-nogit # The entrypoint copies the built JAR to /output (bind-mounted from host). RUN mkdir -p /output CMD ["sh", "-c", "cp spark/target/comet-spark-spark3.5_2.12-*-SNAPSHOT.jar /output/ && echo 'Comet JAR copied to /output/' && ls -lh /output/*.jar"] ================================================ FILE: benchmarks/tpc/infra/docker/docker-compose-laptop.yml ================================================ # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Lightweight Spark standalone cluster for TPC benchmarks on a laptop. # # Single worker, ~12 GB total memory. Suitable for SF1-SF10 testing. # # Usage: # export COMET_JAR=/path/to/comet-spark-0.10.0.jar # docker compose -f benchmarks/tpc/infra/docker/docker-compose-laptop.yml up -d # # Environment variables (set in .env or export before running): # BENCH_IMAGE - Docker image to use (default: comet-bench) # DATA_DIR - Host path to TPC data (default: /tmp/tpc-data) # RESULTS_DIR - Host path for results output (default: /tmp/bench-results) # COMET_JAR - Host path to Comet JAR # GLUTEN_JAR - Host path to Gluten JAR # ICEBERG_JAR - Host path to Iceberg Spark runtime JAR # BENCH_JAVA_HOME - Java home inside container (default: /usr/lib/jvm/java-17-openjdk) # Set to /usr/lib/jvm/java-8-openjdk for Gluten # ASYNC_PROFILER_HOME - async-profiler install path (default: /opt/async-profiler) x-volumes: &volumes - ${DATA_DIR:-/tmp/tpc-data}:/data:ro - ${RESULTS_DIR:-/tmp/bench-results}:/results - ${COMET_JAR:-/dev/null}:/jars/comet.jar:ro - ${GLUTEN_JAR:-/dev/null}:/jars/gluten.jar:ro - ${ICEBERG_JAR:-/dev/null}:/jars/iceberg.jar:ro - ${RESULTS_DIR:-/tmp/bench-results}/logs:/opt/spark/logs - ${RESULTS_DIR:-/tmp/bench-results}/work:/opt/spark/work services: spark-master: image: ${BENCH_IMAGE:-comet-bench} container_name: spark-master hostname: spark-master command: /opt/spark/sbin/start-master.sh --host spark-master ports: - "7077:7077" - "8080:8080" volumes: *volumes environment: - JAVA_HOME=${BENCH_JAVA_HOME:-/usr/lib/jvm/java-17-openjdk} - SPARK_MASTER_HOST=spark-master - SPARK_NO_DAEMONIZE=true spark-worker-1: image: ${BENCH_IMAGE:-comet-bench} container_name: spark-worker-1 hostname: spark-worker-1 depends_on: - spark-master command: /opt/spark/sbin/start-worker.sh spark://spark-master:7077 ports: - "8081:8081" volumes: *volumes environment: - JAVA_HOME=${BENCH_JAVA_HOME:-/usr/lib/jvm/java-17-openjdk} - SPARK_WORKER_CORES=4 - SPARK_WORKER_MEMORY=4g - SPARK_NO_DAEMONIZE=true mem_limit: 8g memswap_limit: 8g stop_grace_period: 30s bench: image: ${BENCH_IMAGE:-comet-bench} container_name: bench-runner depends_on: - spark-master - spark-worker-1 # Override 'command' to run a specific benchmark, e.g.: # docker compose run bench python3 /opt/benchmarks/run.py \ # --engine comet --benchmark tpch --no-restart command: ["echo", "Use 'docker compose run bench python3 /opt/benchmarks/run.py ...' to run benchmarks"] volumes: *volumes environment: - JAVA_HOME=${BENCH_JAVA_HOME:-/usr/lib/jvm/java-17-openjdk} - SPARK_HOME=/opt/spark - SPARK_MASTER=spark://spark-master:7077 - COMET_JAR=/jars/comet.jar - GLUTEN_JAR=/jars/gluten.jar - ICEBERG_JAR=/jars/iceberg.jar - TPCH_DATA=/data - TPCDS_DATA=/data - SPARK_EVENT_LOG_DIR=/results/spark-events - ASYNC_PROFILER_HOME=/opt/async-profiler mem_limit: 4g memswap_limit: 4g ================================================ FILE: benchmarks/tpc/infra/docker/docker-compose.yml ================================================ # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Spark standalone cluster for TPC benchmarks. # # Two workers are used so that shuffles go through the network stack, # which better reflects real cluster behavior. # # Usage: # export COMET_JAR=/path/to/comet-spark-0.10.0.jar # docker compose -f benchmarks/tpc/infra/docker/docker-compose.yml up -d # # Environment variables (set in .env or export before running): # BENCH_IMAGE - Docker image to use (default: comet-bench) # DATA_DIR - Host path to TPC data (default: /tmp/tpc-data) # RESULTS_DIR - Host path for results output (default: /tmp/bench-results) # COMET_JAR - Host path to Comet JAR # GLUTEN_JAR - Host path to Gluten JAR # ICEBERG_JAR - Host path to Iceberg Spark runtime JAR # WORKER_MEM_LIMIT - Hard memory limit per worker container (default: 32g) # BENCH_MEM_LIMIT - Hard memory limit for the bench runner (default: 10g) # BENCH_JAVA_HOME - Java home inside container (default: /usr/lib/jvm/java-17-openjdk) # Set to /usr/lib/jvm/java-8-openjdk for Gluten # ASYNC_PROFILER_HOME - async-profiler install path (default: /opt/async-profiler) x-volumes: &volumes - ${DATA_DIR:-/tmp/tpc-data}:/data:ro - ${RESULTS_DIR:-/tmp/bench-results}:/results - ${COMET_JAR:-/dev/null}:/jars/comet.jar:ro - ${GLUTEN_JAR:-/dev/null}:/jars/gluten.jar:ro - ${ICEBERG_JAR:-/dev/null}:/jars/iceberg.jar:ro - ${RESULTS_DIR:-/tmp/bench-results}/logs:/opt/spark/logs - ${RESULTS_DIR:-/tmp/bench-results}/work:/opt/spark/work x-worker: &worker image: ${BENCH_IMAGE:-comet-bench} depends_on: - spark-master command: /opt/spark/sbin/start-worker.sh spark://spark-master:7077 volumes: *volumes environment: - JAVA_HOME=${BENCH_JAVA_HOME:-/usr/lib/jvm/java-17-openjdk} - SPARK_WORKER_CORES=${WORKER_CORES:-8} - SPARK_WORKER_MEMORY=${WORKER_MEMORY:-16g} - SPARK_NO_DAEMONIZE=true mem_limit: ${WORKER_MEM_LIMIT:-32g} memswap_limit: ${WORKER_MEM_LIMIT:-32g} stop_grace_period: 30s services: spark-master: image: ${BENCH_IMAGE:-comet-bench} container_name: spark-master hostname: spark-master command: /opt/spark/sbin/start-master.sh --host spark-master ports: - "7077:7077" - "8080:8080" volumes: *volumes environment: - JAVA_HOME=${BENCH_JAVA_HOME:-/usr/lib/jvm/java-17-openjdk} - SPARK_MASTER_HOST=spark-master - SPARK_NO_DAEMONIZE=true spark-worker-1: <<: *worker container_name: spark-worker-1 hostname: spark-worker-1 ports: - "8081:8081" spark-worker-2: <<: *worker container_name: spark-worker-2 hostname: spark-worker-2 ports: - "8082:8081" bench: image: ${BENCH_IMAGE:-comet-bench} container_name: bench-runner depends_on: - spark-master - spark-worker-1 - spark-worker-2 # Override 'command' to run a specific benchmark, e.g.: # docker compose run bench python3 /opt/benchmarks/run.py \ # --engine comet --benchmark tpch --no-restart command: ["echo", "Use 'docker compose run bench python3 /opt/benchmarks/run.py ...' to run benchmarks"] volumes: *volumes environment: - JAVA_HOME=${BENCH_JAVA_HOME:-/usr/lib/jvm/java-17-openjdk} - SPARK_HOME=/opt/spark - SPARK_MASTER=spark://spark-master:7077 - COMET_JAR=/jars/comet.jar - GLUTEN_JAR=/jars/gluten.jar - ICEBERG_JAR=/jars/iceberg.jar - TPCH_DATA=/data - TPCDS_DATA=/data - SPARK_EVENT_LOG_DIR=/results/spark-events - ASYNC_PROFILER_HOME=/opt/async-profiler mem_limit: ${BENCH_MEM_LIMIT:-10g} memswap_limit: ${BENCH_MEM_LIMIT:-10g} # Uncomment to enable the Spark History Server for inspecting completed # benchmark runs at http://localhost:18080. Requires event logs in # $RESULTS_DIR/spark-events (created by `mkdir -p $RESULTS_DIR/spark-events` # before starting the cluster). # # history-server: # image: ${BENCH_IMAGE:-comet-bench} # container_name: spark-history # hostname: spark-history # command: /opt/spark/sbin/start-history-server.sh # ports: # - "18080:18080" # volumes: # - ${RESULTS_DIR:-/tmp/bench-results}:/results:ro # environment: # - JAVA_HOME=${BENCH_JAVA_HOME:-/usr/lib/jvm/java-17-openjdk} # - SPARK_HISTORY_OPTS=-Dspark.history.fs.logDirectory=/results/spark-events # - SPARK_NO_DAEMONIZE=true ================================================ FILE: benchmarks/tpc/queries/tpcds/q1.sql ================================================ -- CometBench-DS query 1 derived from TPC-DS query 1 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. with customer_total_return as (select sr_customer_sk as ctr_customer_sk ,sr_store_sk as ctr_store_sk ,sum(SR_RETURN_AMT_INC_TAX) as ctr_total_return from store_returns ,date_dim where sr_returned_date_sk = d_date_sk and d_year =1999 group by sr_customer_sk ,sr_store_sk) select c_customer_id from customer_total_return ctr1 ,store ,customer where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 from customer_total_return ctr2 where ctr1.ctr_store_sk = ctr2.ctr_store_sk) and s_store_sk = ctr1.ctr_store_sk and s_state = 'TN' and ctr1.ctr_customer_sk = c_customer_sk order by c_customer_id LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q10.sql ================================================ -- CometBench-DS query 10 derived from TPC-DS query 10 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select cd_gender, cd_marital_status, cd_education_status, count(*) cnt1, cd_purchase_estimate, count(*) cnt2, cd_credit_rating, count(*) cnt3, cd_dep_count, count(*) cnt4, cd_dep_employed_count, count(*) cnt5, cd_dep_college_count, count(*) cnt6 from customer c,customer_address ca,customer_demographics where c.c_current_addr_sk = ca.ca_address_sk and ca_county in ('Clinton County','Platte County','Franklin County','Louisa County','Harmon County') and cd_demo_sk = c.c_current_cdemo_sk and exists (select * from store_sales,date_dim where c.c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year = 2002 and d_moy between 3 and 3+3) and (exists (select * from web_sales,date_dim where c.c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year = 2002 and d_moy between 3 ANd 3+3) or exists (select * from catalog_sales,date_dim where c.c_customer_sk = cs_ship_customer_sk and cs_sold_date_sk = d_date_sk and d_year = 2002 and d_moy between 3 and 3+3)) group by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating, cd_dep_count, cd_dep_employed_count, cd_dep_college_count order by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating, cd_dep_count, cd_dep_employed_count, cd_dep_college_count LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q11.sql ================================================ -- CometBench-DS query 11 derived from TPC-DS query 11 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. with year_total as ( select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum(ss_ext_list_price-ss_ext_discount_amt) year_total ,'s' sale_type from customer ,store_sales ,date_dim where c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum(ws_ext_list_price-ws_ext_discount_amt) year_total ,'w' sale_type from customer ,web_sales ,date_dim where c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year ) select t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_email_address from year_total t_s_firstyear ,year_total t_s_secyear ,year_total t_w_firstyear ,year_total t_w_secyear where t_s_secyear.customer_id = t_s_firstyear.customer_id and t_s_firstyear.customer_id = t_w_secyear.customer_id and t_s_firstyear.customer_id = t_w_firstyear.customer_id and t_s_firstyear.sale_type = 's' and t_w_firstyear.sale_type = 'w' and t_s_secyear.sale_type = 's' and t_w_secyear.sale_type = 'w' and t_s_firstyear.dyear = 1999 and t_s_secyear.dyear = 1999+1 and t_w_firstyear.dyear = 1999 and t_w_secyear.dyear = 1999+1 and t_s_firstyear.year_total > 0 and t_w_firstyear.year_total > 0 and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else 0.0 end > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else 0.0 end order by t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_email_address LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q12.sql ================================================ -- CometBench-DS query 12 derived from TPC-DS query 12 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price ,sum(ws_ext_sales_price) as itemrevenue ,sum(ws_ext_sales_price)*100/sum(sum(ws_ext_sales_price)) over (partition by i_class) as revenueratio from web_sales ,item ,date_dim where ws_item_sk = i_item_sk and i_category in ('Jewelry', 'Books', 'Women') and ws_sold_date_sk = d_date_sk and d_date between cast('2002-03-22' as date) and (cast('2002-03-22' as date) + INTERVAL '30 DAYS') group by i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price order by i_category ,i_class ,i_item_id ,i_item_desc ,revenueratio LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q13.sql ================================================ -- CometBench-DS query 13 derived from TPC-DS query 13 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select avg(ss_quantity) ,avg(ss_ext_sales_price) ,avg(ss_ext_wholesale_cost) ,sum(ss_ext_wholesale_cost) from store_sales ,store ,customer_demographics ,household_demographics ,customer_address ,date_dim where s_store_sk = ss_store_sk and ss_sold_date_sk = d_date_sk and d_year = 2001 and((ss_hdemo_sk=hd_demo_sk and cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'U' and cd_education_status = '4 yr Degree' and ss_sales_price between 100.00 and 150.00 and hd_dep_count = 3 )or (ss_hdemo_sk=hd_demo_sk and cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'S' and cd_education_status = 'Unknown' and ss_sales_price between 50.00 and 100.00 and hd_dep_count = 1 ) or (ss_hdemo_sk=hd_demo_sk and cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'D' and cd_education_status = '2 yr Degree' and ss_sales_price between 150.00 and 200.00 and hd_dep_count = 1 )) and((ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('CO', 'MI', 'MN') and ss_net_profit between 100 and 200 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('NC', 'NY', 'TX') and ss_net_profit between 150 and 300 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('CA', 'NE', 'TN') and ss_net_profit between 50 and 250 )) ; ================================================ FILE: benchmarks/tpc/queries/tpcds/q14.sql ================================================ -- CometBench-DS query 14 derived from TPC-DS query 14 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. with cross_items as (select i_item_sk ss_item_sk from item, (select iss.i_brand_id brand_id ,iss.i_class_id class_id ,iss.i_category_id category_id from store_sales ,item iss ,date_dim d1 where ss_item_sk = iss.i_item_sk and ss_sold_date_sk = d1.d_date_sk and d1.d_year between 1999 AND 1999 + 2 intersect select ics.i_brand_id ,ics.i_class_id ,ics.i_category_id from catalog_sales ,item ics ,date_dim d2 where cs_item_sk = ics.i_item_sk and cs_sold_date_sk = d2.d_date_sk and d2.d_year between 1999 AND 1999 + 2 intersect select iws.i_brand_id ,iws.i_class_id ,iws.i_category_id from web_sales ,item iws ,date_dim d3 where ws_item_sk = iws.i_item_sk and ws_sold_date_sk = d3.d_date_sk and d3.d_year between 1999 AND 1999 + 2) where i_brand_id = brand_id and i_class_id = class_id and i_category_id = category_id ), avg_sales as (select avg(quantity*list_price) average_sales from (select ss_quantity quantity ,ss_list_price list_price from store_sales ,date_dim where ss_sold_date_sk = d_date_sk and d_year between 1999 and 1999 + 2 union all select cs_quantity quantity ,cs_list_price list_price from catalog_sales ,date_dim where cs_sold_date_sk = d_date_sk and d_year between 1999 and 1999 + 2 union all select ws_quantity quantity ,ws_list_price list_price from web_sales ,date_dim where ws_sold_date_sk = d_date_sk and d_year between 1999 and 1999 + 2) x) select channel, i_brand_id,i_class_id,i_category_id,sum(sales), sum(number_sales) from( select 'store' channel, i_brand_id,i_class_id ,i_category_id,sum(ss_quantity*ss_list_price) sales , count(*) number_sales from store_sales ,item ,date_dim where ss_item_sk in (select ss_item_sk from cross_items) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 1999+2 and d_moy = 11 group by i_brand_id,i_class_id,i_category_id having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales) union all select 'catalog' channel, i_brand_id,i_class_id,i_category_id, sum(cs_quantity*cs_list_price) sales, count(*) number_sales from catalog_sales ,item ,date_dim where cs_item_sk in (select ss_item_sk from cross_items) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 1999+2 and d_moy = 11 group by i_brand_id,i_class_id,i_category_id having sum(cs_quantity*cs_list_price) > (select average_sales from avg_sales) union all select 'web' channel, i_brand_id,i_class_id,i_category_id, sum(ws_quantity*ws_list_price) sales , count(*) number_sales from web_sales ,item ,date_dim where ws_item_sk in (select ss_item_sk from cross_items) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 1999+2 and d_moy = 11 group by i_brand_id,i_class_id,i_category_id having sum(ws_quantity*ws_list_price) > (select average_sales from avg_sales) ) y group by rollup (channel, i_brand_id,i_class_id,i_category_id) order by channel,i_brand_id,i_class_id,i_category_id LIMIT 100; with cross_items as (select i_item_sk ss_item_sk from item, (select iss.i_brand_id brand_id ,iss.i_class_id class_id ,iss.i_category_id category_id from store_sales ,item iss ,date_dim d1 where ss_item_sk = iss.i_item_sk and ss_sold_date_sk = d1.d_date_sk and d1.d_year between 1999 AND 1999 + 2 intersect select ics.i_brand_id ,ics.i_class_id ,ics.i_category_id from catalog_sales ,item ics ,date_dim d2 where cs_item_sk = ics.i_item_sk and cs_sold_date_sk = d2.d_date_sk and d2.d_year between 1999 AND 1999 + 2 intersect select iws.i_brand_id ,iws.i_class_id ,iws.i_category_id from web_sales ,item iws ,date_dim d3 where ws_item_sk = iws.i_item_sk and ws_sold_date_sk = d3.d_date_sk and d3.d_year between 1999 AND 1999 + 2) x where i_brand_id = brand_id and i_class_id = class_id and i_category_id = category_id ), avg_sales as (select avg(quantity*list_price) average_sales from (select ss_quantity quantity ,ss_list_price list_price from store_sales ,date_dim where ss_sold_date_sk = d_date_sk and d_year between 1999 and 1999 + 2 union all select cs_quantity quantity ,cs_list_price list_price from catalog_sales ,date_dim where cs_sold_date_sk = d_date_sk and d_year between 1999 and 1999 + 2 union all select ws_quantity quantity ,ws_list_price list_price from web_sales ,date_dim where ws_sold_date_sk = d_date_sk and d_year between 1999 and 1999 + 2) x) select this_year.channel ty_channel ,this_year.i_brand_id ty_brand ,this_year.i_class_id ty_class ,this_year.i_category_id ty_category ,this_year.sales ty_sales ,this_year.number_sales ty_number_sales ,last_year.channel ly_channel ,last_year.i_brand_id ly_brand ,last_year.i_class_id ly_class ,last_year.i_category_id ly_category ,last_year.sales ly_sales ,last_year.number_sales ly_number_sales from (select 'store' channel, i_brand_id,i_class_id,i_category_id ,sum(ss_quantity*ss_list_price) sales, count(*) number_sales from store_sales ,item ,date_dim where ss_item_sk in (select ss_item_sk from cross_items) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_week_seq = (select d_week_seq from date_dim where d_year = 1999 + 1 and d_moy = 12 and d_dom = 14) group by i_brand_id,i_class_id,i_category_id having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales)) this_year, (select 'store' channel, i_brand_id,i_class_id ,i_category_id, sum(ss_quantity*ss_list_price) sales, count(*) number_sales from store_sales ,item ,date_dim where ss_item_sk in (select ss_item_sk from cross_items) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_week_seq = (select d_week_seq from date_dim where d_year = 1999 and d_moy = 12 and d_dom = 14) group by i_brand_id,i_class_id,i_category_id having sum(ss_quantity*ss_list_price) > (select average_sales from avg_sales)) last_year where this_year.i_brand_id= last_year.i_brand_id and this_year.i_class_id = last_year.i_class_id and this_year.i_category_id = last_year.i_category_id order by this_year.channel, this_year.i_brand_id, this_year.i_class_id, this_year.i_category_id LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q15.sql ================================================ -- CometBench-DS query 15 derived from TPC-DS query 15 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select ca_zip ,sum(cs_sales_price) from catalog_sales ,customer ,customer_address ,date_dim where cs_bill_customer_sk = c_customer_sk and c_current_addr_sk = ca_address_sk and ( substr(ca_zip,1,5) in ('85669', '86197','88274','83405','86475', '85392', '85460', '80348', '81792') or ca_state in ('CA','WA','GA') or cs_sales_price > 500) and cs_sold_date_sk = d_date_sk and d_qoy = 2 and d_year = 2002 group by ca_zip order by ca_zip LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q16.sql ================================================ -- CometBench-DS query 16 derived from TPC-DS query 16 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select count(distinct cs_order_number) as `order count` ,sum(cs_ext_ship_cost) as `total shipping cost` ,sum(cs_net_profit) as `total net profit` from catalog_sales cs1 ,date_dim ,customer_address ,call_center where d_date between '1999-5-01' and (cast('1999-5-01' as date) + INTERVAL '60 DAYS') and cs1.cs_ship_date_sk = d_date_sk and cs1.cs_ship_addr_sk = ca_address_sk and ca_state = 'ID' and cs1.cs_call_center_sk = cc_call_center_sk and cc_county in ('Williamson County','Williamson County','Williamson County','Williamson County', 'Williamson County' ) and exists (select * from catalog_sales cs2 where cs1.cs_order_number = cs2.cs_order_number and cs1.cs_warehouse_sk <> cs2.cs_warehouse_sk) and not exists(select * from catalog_returns cr1 where cs1.cs_order_number = cr1.cr_order_number) order by count(distinct cs_order_number) LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q17.sql ================================================ -- CometBench-DS query 17 derived from TPC-DS query 17 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select i_item_id ,i_item_desc ,s_state ,count(ss_quantity) as store_sales_quantitycount ,avg(ss_quantity) as store_sales_quantityave ,stddev_samp(ss_quantity) as store_sales_quantitystdev ,stddev_samp(ss_quantity)/avg(ss_quantity) as store_sales_quantitycov ,count(sr_return_quantity) as store_returns_quantitycount ,avg(sr_return_quantity) as store_returns_quantityave ,stddev_samp(sr_return_quantity) as store_returns_quantitystdev ,stddev_samp(sr_return_quantity)/avg(sr_return_quantity) as store_returns_quantitycov ,count(cs_quantity) as catalog_sales_quantitycount ,avg(cs_quantity) as catalog_sales_quantityave ,stddev_samp(cs_quantity) as catalog_sales_quantitystdev ,stddev_samp(cs_quantity)/avg(cs_quantity) as catalog_sales_quantitycov from store_sales ,store_returns ,catalog_sales ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,item where d1.d_quarter_name = '1999Q1' and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and ss_customer_sk = sr_customer_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and sr_returned_date_sk = d2.d_date_sk and d2.d_quarter_name in ('1999Q1','1999Q2','1999Q3') and sr_customer_sk = cs_bill_customer_sk and sr_item_sk = cs_item_sk and cs_sold_date_sk = d3.d_date_sk and d3.d_quarter_name in ('1999Q1','1999Q2','1999Q3') group by i_item_id ,i_item_desc ,s_state order by i_item_id ,i_item_desc ,s_state LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q18.sql ================================================ -- CometBench-DS query 18 derived from TPC-DS query 18 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select i_item_id, ca_country, ca_state, ca_county, avg( cast(cs_quantity as decimal(12,2))) agg1, avg( cast(cs_list_price as decimal(12,2))) agg2, avg( cast(cs_coupon_amt as decimal(12,2))) agg3, avg( cast(cs_sales_price as decimal(12,2))) agg4, avg( cast(cs_net_profit as decimal(12,2))) agg5, avg( cast(c_birth_year as decimal(12,2))) agg6, avg( cast(cd1.cd_dep_count as decimal(12,2))) agg7 from catalog_sales, customer_demographics cd1, customer_demographics cd2, customer, customer_address, date_dim, item where cs_sold_date_sk = d_date_sk and cs_item_sk = i_item_sk and cs_bill_cdemo_sk = cd1.cd_demo_sk and cs_bill_customer_sk = c_customer_sk and cd1.cd_gender = 'M' and cd1.cd_education_status = 'Primary' and c_current_cdemo_sk = cd2.cd_demo_sk and c_current_addr_sk = ca_address_sk and c_birth_month in (1,2,9,5,11,3) and d_year = 1998 and ca_state in ('MS','NE','IA' ,'MI','GA','NY','CO') group by rollup (i_item_id, ca_country, ca_state, ca_county) order by ca_country, ca_state, ca_county, i_item_id LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q19.sql ================================================ -- CometBench-DS query 19 derived from TPC-DS query 19 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select i_brand_id brand_id, i_brand brand, i_manufact_id, i_manufact, sum(ss_ext_sales_price) ext_price from date_dim, store_sales, item,customer,customer_address,store where d_date_sk = ss_sold_date_sk and ss_item_sk = i_item_sk and i_manager_id=8 and d_moy=11 and d_year=1999 and ss_customer_sk = c_customer_sk and c_current_addr_sk = ca_address_sk and substr(ca_zip,1,5) <> substr(s_zip,1,5) and ss_store_sk = s_store_sk group by i_brand ,i_brand_id ,i_manufact_id ,i_manufact order by ext_price desc ,i_brand ,i_brand_id ,i_manufact_id ,i_manufact LIMIT 100 ; ================================================ FILE: benchmarks/tpc/queries/tpcds/q2.sql ================================================ -- CometBench-DS query 2 derived from TPC-DS query 2 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. with wscs as (select sold_date_sk ,sales_price from (select ws_sold_date_sk sold_date_sk ,ws_ext_sales_price sales_price from web_sales union all select cs_sold_date_sk sold_date_sk ,cs_ext_sales_price sales_price from catalog_sales)), wswscs as (select d_week_seq, sum(case when (d_day_name='Sunday') then sales_price else null end) sun_sales, sum(case when (d_day_name='Monday') then sales_price else null end) mon_sales, sum(case when (d_day_name='Tuesday') then sales_price else null end) tue_sales, sum(case when (d_day_name='Wednesday') then sales_price else null end) wed_sales, sum(case when (d_day_name='Thursday') then sales_price else null end) thu_sales, sum(case when (d_day_name='Friday') then sales_price else null end) fri_sales, sum(case when (d_day_name='Saturday') then sales_price else null end) sat_sales from wscs ,date_dim where d_date_sk = sold_date_sk group by d_week_seq) select d_week_seq1 ,round(sun_sales1/sun_sales2,2) ,round(mon_sales1/mon_sales2,2) ,round(tue_sales1/tue_sales2,2) ,round(wed_sales1/wed_sales2,2) ,round(thu_sales1/thu_sales2,2) ,round(fri_sales1/fri_sales2,2) ,round(sat_sales1/sat_sales2,2) from (select wswscs.d_week_seq d_week_seq1 ,sun_sales sun_sales1 ,mon_sales mon_sales1 ,tue_sales tue_sales1 ,wed_sales wed_sales1 ,thu_sales thu_sales1 ,fri_sales fri_sales1 ,sat_sales sat_sales1 from wswscs,date_dim where date_dim.d_week_seq = wswscs.d_week_seq and d_year = 2000) y, (select wswscs.d_week_seq d_week_seq2 ,sun_sales sun_sales2 ,mon_sales mon_sales2 ,tue_sales tue_sales2 ,wed_sales wed_sales2 ,thu_sales thu_sales2 ,fri_sales fri_sales2 ,sat_sales sat_sales2 from wswscs ,date_dim where date_dim.d_week_seq = wswscs.d_week_seq and d_year = 2000+1) z where d_week_seq1=d_week_seq2-53 order by d_week_seq1; ================================================ FILE: benchmarks/tpc/queries/tpcds/q20.sql ================================================ -- CometBench-DS query 20 derived from TPC-DS query 20 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price ,sum(cs_ext_sales_price) as itemrevenue ,sum(cs_ext_sales_price)*100/sum(sum(cs_ext_sales_price)) over (partition by i_class) as revenueratio from catalog_sales ,item ,date_dim where cs_item_sk = i_item_sk and i_category in ('Children', 'Sports', 'Music') and cs_sold_date_sk = d_date_sk and d_date between cast('2002-04-01' as date) and (cast('2002-04-01' as date) + INTERVAL '30 DAYS') group by i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price order by i_category ,i_class ,i_item_id ,i_item_desc ,revenueratio LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q21.sql ================================================ -- CometBench-DS query 21 derived from TPC-DS query 21 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select * from(select w_warehouse_name ,i_item_id ,sum(case when (cast(d_date as date) < cast ('2000-05-19' as date)) then inv_quantity_on_hand else 0 end) as inv_before ,sum(case when (cast(d_date as date) >= cast ('2000-05-19' as date)) then inv_quantity_on_hand else 0 end) as inv_after from inventory ,warehouse ,item ,date_dim where i_current_price between 0.99 and 1.49 and i_item_sk = inv_item_sk and inv_warehouse_sk = w_warehouse_sk and inv_date_sk = d_date_sk and d_date between (cast ('2000-05-19' as date) - INTERVAL '30 DAYS') and (cast ('2000-05-19' as date) + INTERVAL '30 DAYS') group by w_warehouse_name, i_item_id) x where (case when inv_before > 0 then inv_after / inv_before else null end) between 2.0/3.0 and 3.0/2.0 order by w_warehouse_name ,i_item_id LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q22.sql ================================================ -- CometBench-DS query 22 derived from TPC-DS query 22 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select i_product_name ,i_brand ,i_class ,i_category ,avg(inv_quantity_on_hand) qoh from inventory ,date_dim ,item where inv_date_sk=d_date_sk and inv_item_sk=i_item_sk and d_month_seq between 1201 and 1201 + 11 group by rollup(i_product_name ,i_brand ,i_class ,i_category) order by qoh, i_product_name, i_brand, i_class, i_category LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q23.sql ================================================ -- CometBench-DS query 23 derived from TPC-DS query 23 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. with frequent_ss_items as (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt from store_sales ,date_dim ,item where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and d_year in (2000,2000+1,2000+2,2000+3) group by substr(i_item_desc,1,30),i_item_sk,d_date having count(*) >4), max_store_sales as (select max(csales) tpcds_cmax from (select c_customer_sk,sum(ss_quantity*ss_sales_price) csales from store_sales ,customer ,date_dim where ss_customer_sk = c_customer_sk and ss_sold_date_sk = d_date_sk and d_year in (2000,2000+1,2000+2,2000+3) group by c_customer_sk)), best_ss_customer as (select c_customer_sk,sum(ss_quantity*ss_sales_price) ssales from store_sales ,customer where ss_customer_sk = c_customer_sk group by c_customer_sk having sum(ss_quantity*ss_sales_price) > (95/100.0) * (select * from max_store_sales)) select sum(sales) from (select cs_quantity*cs_list_price sales from catalog_sales ,date_dim where d_year = 2000 and d_moy = 3 and cs_sold_date_sk = d_date_sk and cs_item_sk in (select item_sk from frequent_ss_items) and cs_bill_customer_sk in (select c_customer_sk from best_ss_customer) union all select ws_quantity*ws_list_price sales from web_sales ,date_dim where d_year = 2000 and d_moy = 3 and ws_sold_date_sk = d_date_sk and ws_item_sk in (select item_sk from frequent_ss_items) and ws_bill_customer_sk in (select c_customer_sk from best_ss_customer)) LIMIT 100; with frequent_ss_items as (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt from store_sales ,date_dim ,item where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and d_year in (2000,2000 + 1,2000 + 2,2000 + 3) group by substr(i_item_desc,1,30),i_item_sk,d_date having count(*) >4), max_store_sales as (select max(csales) tpcds_cmax from (select c_customer_sk,sum(ss_quantity*ss_sales_price) csales from store_sales ,customer ,date_dim where ss_customer_sk = c_customer_sk and ss_sold_date_sk = d_date_sk and d_year in (2000,2000+1,2000+2,2000+3) group by c_customer_sk)), best_ss_customer as (select c_customer_sk,sum(ss_quantity*ss_sales_price) ssales from store_sales ,customer where ss_customer_sk = c_customer_sk group by c_customer_sk having sum(ss_quantity*ss_sales_price) > (95/100.0) * (select * from max_store_sales)) select c_last_name,c_first_name,sales from (select c_last_name,c_first_name,sum(cs_quantity*cs_list_price) sales from catalog_sales ,customer ,date_dim where d_year = 2000 and d_moy = 3 and cs_sold_date_sk = d_date_sk and cs_item_sk in (select item_sk from frequent_ss_items) and cs_bill_customer_sk in (select c_customer_sk from best_ss_customer) and cs_bill_customer_sk = c_customer_sk group by c_last_name,c_first_name union all select c_last_name,c_first_name,sum(ws_quantity*ws_list_price) sales from web_sales ,customer ,date_dim where d_year = 2000 and d_moy = 3 and ws_sold_date_sk = d_date_sk and ws_item_sk in (select item_sk from frequent_ss_items) and ws_bill_customer_sk in (select c_customer_sk from best_ss_customer) and ws_bill_customer_sk = c_customer_sk group by c_last_name,c_first_name) order by c_last_name,c_first_name,sales LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q24.sql ================================================ -- CometBench-DS query 24 derived from TPC-DS query 24 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. with ssales as (select c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size ,sum(ss_net_profit) netpaid from store_sales ,store_returns ,store ,item ,customer ,customer_address where ss_ticket_number = sr_ticket_number and ss_item_sk = sr_item_sk and ss_customer_sk = c_customer_sk and ss_item_sk = i_item_sk and ss_store_sk = s_store_sk and c_current_addr_sk = ca_address_sk and c_birth_country <> upper(ca_country) and s_zip = ca_zip and s_market_id=10 group by c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size) select c_last_name ,c_first_name ,s_store_name ,sum(netpaid) paid from ssales where i_color = 'orchid' group by c_last_name ,c_first_name ,s_store_name having sum(netpaid) > (select 0.05*avg(netpaid) from ssales) order by c_last_name ,c_first_name ,s_store_name ; with ssales as (select c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size ,sum(ss_net_profit) netpaid from store_sales ,store_returns ,store ,item ,customer ,customer_address where ss_ticket_number = sr_ticket_number and ss_item_sk = sr_item_sk and ss_customer_sk = c_customer_sk and ss_item_sk = i_item_sk and ss_store_sk = s_store_sk and c_current_addr_sk = ca_address_sk and c_birth_country <> upper(ca_country) and s_zip = ca_zip and s_market_id = 10 group by c_last_name ,c_first_name ,s_store_name ,ca_state ,s_state ,i_color ,i_current_price ,i_manager_id ,i_units ,i_size) select c_last_name ,c_first_name ,s_store_name ,sum(netpaid) paid from ssales where i_color = 'green' group by c_last_name ,c_first_name ,s_store_name having sum(netpaid) > (select 0.05*avg(netpaid) from ssales) order by c_last_name ,c_first_name ,s_store_name ; ================================================ FILE: benchmarks/tpc/queries/tpcds/q25.sql ================================================ -- CometBench-DS query 25 derived from TPC-DS query 25 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select i_item_id ,i_item_desc ,s_store_id ,s_store_name ,min(ss_net_profit) as store_sales_profit ,min(sr_net_loss) as store_returns_loss ,min(cs_net_profit) as catalog_sales_profit from store_sales ,store_returns ,catalog_sales ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,item where d1.d_moy = 4 and d1.d_year = 2002 and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and ss_customer_sk = sr_customer_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and sr_returned_date_sk = d2.d_date_sk and d2.d_moy between 4 and 10 and d2.d_year = 2002 and sr_customer_sk = cs_bill_customer_sk and sr_item_sk = cs_item_sk and cs_sold_date_sk = d3.d_date_sk and d3.d_moy between 4 and 10 and d3.d_year = 2002 group by i_item_id ,i_item_desc ,s_store_id ,s_store_name order by i_item_id ,i_item_desc ,s_store_id ,s_store_name LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q26.sql ================================================ -- CometBench-DS query 26 derived from TPC-DS query 26 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select i_item_id, avg(cs_quantity) agg1, avg(cs_list_price) agg2, avg(cs_coupon_amt) agg3, avg(cs_sales_price) agg4 from catalog_sales, customer_demographics, date_dim, item, promotion where cs_sold_date_sk = d_date_sk and cs_item_sk = i_item_sk and cs_bill_cdemo_sk = cd_demo_sk and cs_promo_sk = p_promo_sk and cd_gender = 'F' and cd_marital_status = 'M' and cd_education_status = '4 yr Degree' and (p_channel_email = 'N' or p_channel_event = 'N') and d_year = 2000 group by i_item_id order by i_item_id LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q27.sql ================================================ -- CometBench-DS query 27 derived from TPC-DS query 27 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select i_item_id, s_state, grouping(s_state) g_state, avg(ss_quantity) agg1, avg(ss_list_price) agg2, avg(ss_coupon_amt) agg3, avg(ss_sales_price) agg4 from store_sales, customer_demographics, date_dim, store, item where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and ss_store_sk = s_store_sk and ss_cdemo_sk = cd_demo_sk and cd_gender = 'M' and cd_marital_status = 'U' and cd_education_status = 'Secondary' and d_year = 2000 and s_state in ('TN','TN', 'TN', 'TN', 'TN', 'TN') group by rollup (i_item_id, s_state) order by i_item_id ,s_state LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q28.sql ================================================ -- CometBench-DS query 28 derived from TPC-DS query 28 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select * from (select avg(ss_list_price) B1_LP ,count(ss_list_price) B1_CNT ,count(distinct ss_list_price) B1_CNTD from store_sales where ss_quantity between 0 and 5 and (ss_list_price between 28 and 28+10 or ss_coupon_amt between 12573 and 12573+1000 or ss_wholesale_cost between 33 and 33+20)) B1, (select avg(ss_list_price) B2_LP ,count(ss_list_price) B2_CNT ,count(distinct ss_list_price) B2_CNTD from store_sales where ss_quantity between 6 and 10 and (ss_list_price between 143 and 143+10 or ss_coupon_amt between 5562 and 5562+1000 or ss_wholesale_cost between 45 and 45+20)) B2, (select avg(ss_list_price) B3_LP ,count(ss_list_price) B3_CNT ,count(distinct ss_list_price) B3_CNTD from store_sales where ss_quantity between 11 and 15 and (ss_list_price between 159 and 159+10 or ss_coupon_amt between 2807 and 2807+1000 or ss_wholesale_cost between 24 and 24+20)) B3, (select avg(ss_list_price) B4_LP ,count(ss_list_price) B4_CNT ,count(distinct ss_list_price) B4_CNTD from store_sales where ss_quantity between 16 and 20 and (ss_list_price between 24 and 24+10 or ss_coupon_amt between 3706 and 3706+1000 or ss_wholesale_cost between 46 and 46+20)) B4, (select avg(ss_list_price) B5_LP ,count(ss_list_price) B5_CNT ,count(distinct ss_list_price) B5_CNTD from store_sales where ss_quantity between 21 and 25 and (ss_list_price between 76 and 76+10 or ss_coupon_amt between 2096 and 2096+1000 or ss_wholesale_cost between 50 and 50+20)) B5, (select avg(ss_list_price) B6_LP ,count(ss_list_price) B6_CNT ,count(distinct ss_list_price) B6_CNTD from store_sales where ss_quantity between 26 and 30 and (ss_list_price between 169 and 169+10 or ss_coupon_amt between 10672 and 10672+1000 or ss_wholesale_cost between 58 and 58+20)) B6 LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q29.sql ================================================ -- CometBench-DS query 29 derived from TPC-DS query 29 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select i_item_id ,i_item_desc ,s_store_id ,s_store_name ,stddev_samp(ss_quantity) as store_sales_quantity ,stddev_samp(sr_return_quantity) as store_returns_quantity ,stddev_samp(cs_quantity) as catalog_sales_quantity from store_sales ,store_returns ,catalog_sales ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,item where d1.d_moy = 4 and d1.d_year = 1999 and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and ss_customer_sk = sr_customer_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and sr_returned_date_sk = d2.d_date_sk and d2.d_moy between 4 and 4 + 3 and d2.d_year = 1999 and sr_customer_sk = cs_bill_customer_sk and sr_item_sk = cs_item_sk and cs_sold_date_sk = d3.d_date_sk and d3.d_year in (1999,1999+1,1999+2) group by i_item_id ,i_item_desc ,s_store_id ,s_store_name order by i_item_id ,i_item_desc ,s_store_id ,s_store_name LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q3.sql ================================================ -- CometBench-DS query 3 derived from TPC-DS query 3 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select dt.d_year ,item.i_brand_id brand_id ,item.i_brand brand ,sum(ss_net_profit) sum_agg from date_dim dt ,store_sales ,item where dt.d_date_sk = store_sales.ss_sold_date_sk and store_sales.ss_item_sk = item.i_item_sk and item.i_manufact_id = 445 and dt.d_moy=12 group by dt.d_year ,item.i_brand ,item.i_brand_id order by dt.d_year ,sum_agg desc ,brand_id LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q30.sql ================================================ -- CometBench-DS query 30 derived from TPC-DS query 30 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. with customer_total_return as (select wr_returning_customer_sk as ctr_customer_sk ,ca_state as ctr_state, sum(wr_return_amt) as ctr_total_return from web_returns ,date_dim ,customer_address where wr_returned_date_sk = d_date_sk and d_year =2000 and wr_returning_addr_sk = ca_address_sk group by wr_returning_customer_sk ,ca_state) select c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag ,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address ,c_last_review_date_sk,ctr_total_return from customer_total_return ctr1 ,customer_address ,customer where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 from customer_total_return ctr2 where ctr1.ctr_state = ctr2.ctr_state) and ca_address_sk = c_current_addr_sk and ca_state = 'KS' and ctr1.ctr_customer_sk = c_customer_sk order by c_customer_id,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag ,c_birth_day,c_birth_month,c_birth_year,c_birth_country,c_login,c_email_address ,c_last_review_date_sk,ctr_total_return LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q31.sql ================================================ -- CometBench-DS query 31 derived from TPC-DS query 31 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. with ss as (select ca_county,d_qoy, d_year,sum(ss_ext_sales_price) as store_sales from store_sales,date_dim,customer_address where ss_sold_date_sk = d_date_sk and ss_addr_sk=ca_address_sk group by ca_county,d_qoy, d_year), ws as (select ca_county,d_qoy, d_year,sum(ws_ext_sales_price) as web_sales from web_sales,date_dim,customer_address where ws_sold_date_sk = d_date_sk and ws_bill_addr_sk=ca_address_sk group by ca_county,d_qoy, d_year) select ss1.ca_county ,ss1.d_year ,ws2.web_sales/ws1.web_sales web_q1_q2_increase ,ss2.store_sales/ss1.store_sales store_q1_q2_increase ,ws3.web_sales/ws2.web_sales web_q2_q3_increase ,ss3.store_sales/ss2.store_sales store_q2_q3_increase from ss ss1 ,ss ss2 ,ss ss3 ,ws ws1 ,ws ws2 ,ws ws3 where ss1.d_qoy = 1 and ss1.d_year = 1999 and ss1.ca_county = ss2.ca_county and ss2.d_qoy = 2 and ss2.d_year = 1999 and ss2.ca_county = ss3.ca_county and ss3.d_qoy = 3 and ss3.d_year = 1999 and ss1.ca_county = ws1.ca_county and ws1.d_qoy = 1 and ws1.d_year = 1999 and ws1.ca_county = ws2.ca_county and ws2.d_qoy = 2 and ws2.d_year = 1999 and ws1.ca_county = ws3.ca_county and ws3.d_qoy = 3 and ws3.d_year =1999 and case when ws1.web_sales > 0 then ws2.web_sales/ws1.web_sales else null end > case when ss1.store_sales > 0 then ss2.store_sales/ss1.store_sales else null end and case when ws2.web_sales > 0 then ws3.web_sales/ws2.web_sales else null end > case when ss2.store_sales > 0 then ss3.store_sales/ss2.store_sales else null end order by ss1.ca_county; ================================================ FILE: benchmarks/tpc/queries/tpcds/q32.sql ================================================ -- CometBench-DS query 32 derived from TPC-DS query 32 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select sum(cs_ext_discount_amt) as `excess discount amount` from catalog_sales ,item ,date_dim where i_manufact_id = 283 and i_item_sk = cs_item_sk and d_date between '1999-02-22' and (cast('1999-02-22' as date) + INTERVAL '90 DAYS') and d_date_sk = cs_sold_date_sk and cs_ext_discount_amt > ( select 1.3 * avg(cs_ext_discount_amt) from catalog_sales ,date_dim where cs_item_sk = i_item_sk and d_date between '1999-02-22' and (cast('1999-02-22' as date) + INTERVAL '90 DAYS') and d_date_sk = cs_sold_date_sk ) LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q33.sql ================================================ -- CometBench-DS query 33 derived from TPC-DS query 33 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. with ss as ( select i_manufact_id,sum(ss_ext_sales_price) total_sales from store_sales, date_dim, customer_address, item where i_manufact_id in (select i_manufact_id from item where i_category in ('Books')) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 1999 and d_moy = 4 and ss_addr_sk = ca_address_sk and ca_gmt_offset = -5 group by i_manufact_id), cs as ( select i_manufact_id,sum(cs_ext_sales_price) total_sales from catalog_sales, date_dim, customer_address, item where i_manufact_id in (select i_manufact_id from item where i_category in ('Books')) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 1999 and d_moy = 4 and cs_bill_addr_sk = ca_address_sk and ca_gmt_offset = -5 group by i_manufact_id), ws as ( select i_manufact_id,sum(ws_ext_sales_price) total_sales from web_sales, date_dim, customer_address, item where i_manufact_id in (select i_manufact_id from item where i_category in ('Books')) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 1999 and d_moy = 4 and ws_bill_addr_sk = ca_address_sk and ca_gmt_offset = -5 group by i_manufact_id) select i_manufact_id ,sum(total_sales) total_sales from (select * from ss union all select * from cs union all select * from ws) tmp1 group by i_manufact_id order by total_sales LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q34.sql ================================================ -- CometBench-DS query 34 derived from TPC-DS query 34 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select c_last_name ,c_first_name ,c_salutation ,c_preferred_cust_flag ,ss_ticket_number ,cnt from (select ss_ticket_number ,ss_customer_sk ,count(*) cnt from store_sales,date_dim,store,household_demographics where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and (date_dim.d_dom between 1 and 3 or date_dim.d_dom between 25 and 28) and (household_demographics.hd_buy_potential = '501-1000' or household_demographics.hd_buy_potential = 'Unknown') and household_demographics.hd_vehicle_count > 0 and (case when household_demographics.hd_vehicle_count > 0 then household_demographics.hd_dep_count/ household_demographics.hd_vehicle_count else null end) > 1.2 and date_dim.d_year in (2000,2000+1,2000+2) and store.s_county in ('Williamson County','Williamson County','Williamson County','Williamson County', 'Williamson County','Williamson County','Williamson County','Williamson County') group by ss_ticket_number,ss_customer_sk) dn,customer where ss_customer_sk = c_customer_sk and cnt between 15 and 20 order by c_last_name,c_first_name,c_salutation,c_preferred_cust_flag desc, ss_ticket_number; ================================================ FILE: benchmarks/tpc/queries/tpcds/q35.sql ================================================ -- CometBench-DS query 35 derived from TPC-DS query 35 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select ca_state, cd_gender, cd_marital_status, cd_dep_count, count(*) cnt1, max(cd_dep_count), stddev_samp(cd_dep_count), stddev_samp(cd_dep_count), cd_dep_employed_count, count(*) cnt2, max(cd_dep_employed_count), stddev_samp(cd_dep_employed_count), stddev_samp(cd_dep_employed_count), cd_dep_college_count, count(*) cnt3, max(cd_dep_college_count), stddev_samp(cd_dep_college_count), stddev_samp(cd_dep_college_count) from customer c,customer_address ca,customer_demographics where c.c_current_addr_sk = ca.ca_address_sk and cd_demo_sk = c.c_current_cdemo_sk and exists (select * from store_sales,date_dim where c.c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year = 2000 and d_qoy < 4) and (exists (select * from web_sales,date_dim where c.c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year = 2000 and d_qoy < 4) or exists (select * from catalog_sales,date_dim where c.c_customer_sk = cs_ship_customer_sk and cs_sold_date_sk = d_date_sk and d_year = 2000 and d_qoy < 4)) group by ca_state, cd_gender, cd_marital_status, cd_dep_count, cd_dep_employed_count, cd_dep_college_count order by ca_state, cd_gender, cd_marital_status, cd_dep_count, cd_dep_employed_count, cd_dep_college_count LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q36.sql ================================================ -- CometBench-DS query 36 derived from TPC-DS query 36 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select sum(ss_net_profit)/sum(ss_ext_sales_price) as gross_margin ,i_category ,i_class ,grouping(i_category)+grouping(i_class) as lochierarchy ,rank() over ( partition by grouping(i_category)+grouping(i_class), case when grouping(i_class) = 0 then i_category end order by sum(ss_net_profit)/sum(ss_ext_sales_price) asc) as rank_within_parent from store_sales ,date_dim d1 ,item ,store where d1.d_year = 2001 and d1.d_date_sk = ss_sold_date_sk and i_item_sk = ss_item_sk and s_store_sk = ss_store_sk and s_state in ('TN','TN','TN','TN', 'TN','TN','TN','TN') group by rollup(i_category,i_class) order by lochierarchy desc ,case when lochierarchy = 0 then i_category end ,rank_within_parent LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q37.sql ================================================ -- CometBench-DS query 37 derived from TPC-DS query 37 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select i_item_id ,i_item_desc ,i_current_price from item, inventory, date_dim, catalog_sales where i_current_price between 26 and 26 + 30 and inv_item_sk = i_item_sk and d_date_sk=inv_date_sk and d_date between cast('2001-06-09' as date) and (cast('2001-06-09' as date) + INTERVAL '60 DAYS') and i_manufact_id in (744,884,722,693) and inv_quantity_on_hand between 100 and 500 and cs_item_sk = i_item_sk group by i_item_id,i_item_desc,i_current_price order by i_item_id LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q38.sql ================================================ -- CometBench-DS query 38 derived from TPC-DS query 38 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select count(*) from ( select distinct c_last_name, c_first_name, d_date from store_sales, date_dim, customer where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_customer_sk = customer.c_customer_sk and d_month_seq between 1190 and 1190 + 11 intersect select distinct c_last_name, c_first_name, d_date from catalog_sales, date_dim, customer where catalog_sales.cs_sold_date_sk = date_dim.d_date_sk and catalog_sales.cs_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1190 and 1190 + 11 intersect select distinct c_last_name, c_first_name, d_date from web_sales, date_dim, customer where web_sales.ws_sold_date_sk = date_dim.d_date_sk and web_sales.ws_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1190 and 1190 + 11 ) hot_cust LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q39.sql ================================================ -- CometBench-DS query 39 derived from TPC-DS query 39 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. with inv as (select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stdev,mean, case mean when 0 then null else stdev/mean end cov from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean from inventory ,item ,warehouse ,date_dim where inv_item_sk = i_item_sk and inv_warehouse_sk = w_warehouse_sk and inv_date_sk = d_date_sk and d_year =2001 group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo where case mean when 0 then 0 else stdev/mean end > 1) select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov from inv inv1,inv inv2 where inv1.i_item_sk = inv2.i_item_sk and inv1.w_warehouse_sk = inv2.w_warehouse_sk and inv1.d_moy=1 and inv2.d_moy=1+1 order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov ,inv2.d_moy,inv2.mean, inv2.cov ; with inv as (select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stdev,mean, case mean when 0 then null else stdev/mean end cov from(select w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy ,stddev_samp(inv_quantity_on_hand) stdev,avg(inv_quantity_on_hand) mean from inventory ,item ,warehouse ,date_dim where inv_item_sk = i_item_sk and inv_warehouse_sk = w_warehouse_sk and inv_date_sk = d_date_sk and d_year =2001 group by w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy) foo where case mean when 0 then 0 else stdev/mean end > 1) select inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean, inv1.cov ,inv2.w_warehouse_sk,inv2.i_item_sk,inv2.d_moy,inv2.mean, inv2.cov from inv inv1,inv inv2 where inv1.i_item_sk = inv2.i_item_sk and inv1.w_warehouse_sk = inv2.w_warehouse_sk and inv1.d_moy=1 and inv2.d_moy=1+1 and inv1.cov > 1.5 order by inv1.w_warehouse_sk,inv1.i_item_sk,inv1.d_moy,inv1.mean,inv1.cov ,inv2.d_moy,inv2.mean, inv2.cov ; ================================================ FILE: benchmarks/tpc/queries/tpcds/q4.sql ================================================ -- CometBench-DS query 4 derived from TPC-DS query 4 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. with year_total as ( select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum(((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)/2) year_total ,'s' sale_type from customer ,store_sales ,date_dim where c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum((((cs_ext_list_price-cs_ext_wholesale_cost-cs_ext_discount_amt)+cs_ext_sales_price)/2) ) year_total ,'c' sale_type from customer ,catalog_sales ,date_dim where c_customer_sk = cs_bill_customer_sk and cs_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,c_preferred_cust_flag customer_preferred_cust_flag ,c_birth_country customer_birth_country ,c_login customer_login ,c_email_address customer_email_address ,d_year dyear ,sum((((ws_ext_list_price-ws_ext_wholesale_cost-ws_ext_discount_amt)+ws_ext_sales_price)/2) ) year_total ,'w' sale_type from customer ,web_sales ,date_dim where c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk group by c_customer_id ,c_first_name ,c_last_name ,c_preferred_cust_flag ,c_birth_country ,c_login ,c_email_address ,d_year ) select t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_email_address from year_total t_s_firstyear ,year_total t_s_secyear ,year_total t_c_firstyear ,year_total t_c_secyear ,year_total t_w_firstyear ,year_total t_w_secyear where t_s_secyear.customer_id = t_s_firstyear.customer_id and t_s_firstyear.customer_id = t_c_secyear.customer_id and t_s_firstyear.customer_id = t_c_firstyear.customer_id and t_s_firstyear.customer_id = t_w_firstyear.customer_id and t_s_firstyear.customer_id = t_w_secyear.customer_id and t_s_firstyear.sale_type = 's' and t_c_firstyear.sale_type = 'c' and t_w_firstyear.sale_type = 'w' and t_s_secyear.sale_type = 's' and t_c_secyear.sale_type = 'c' and t_w_secyear.sale_type = 'w' and t_s_firstyear.dyear = 2001 and t_s_secyear.dyear = 2001+1 and t_c_firstyear.dyear = 2001 and t_c_secyear.dyear = 2001+1 and t_w_firstyear.dyear = 2001 and t_w_secyear.dyear = 2001+1 and t_s_firstyear.year_total > 0 and t_c_firstyear.year_total > 0 and t_w_firstyear.year_total > 0 and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end > case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end order by t_s_secyear.customer_id ,t_s_secyear.customer_first_name ,t_s_secyear.customer_last_name ,t_s_secyear.customer_email_address LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q40.sql ================================================ -- CometBench-DS query 40 derived from TPC-DS query 40 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select w_state ,i_item_id ,sum(case when (cast(d_date as date) < cast ('2002-05-18' as date)) then cs_sales_price - coalesce(cr_refunded_cash,0) else 0 end) as sales_before ,sum(case when (cast(d_date as date) >= cast ('2002-05-18' as date)) then cs_sales_price - coalesce(cr_refunded_cash,0) else 0 end) as sales_after from catalog_sales left outer join catalog_returns on (cs_order_number = cr_order_number and cs_item_sk = cr_item_sk) ,warehouse ,item ,date_dim where i_current_price between 0.99 and 1.49 and i_item_sk = cs_item_sk and cs_warehouse_sk = w_warehouse_sk and cs_sold_date_sk = d_date_sk and d_date between (cast ('2002-05-18' as date) - INTERVAL '30 DAYS') and (cast ('2002-05-18' as date) + INTERVAL '30 DAYS') group by w_state,i_item_id order by w_state,i_item_id LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q41.sql ================================================ -- CometBench-DS query 41 derived from TPC-DS query 41 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select distinct(i_product_name) from item i1 where i_manufact_id between 668 and 668+40 and (select count(*) as item_cnt from item where (i_manufact = i1.i_manufact and ((i_category = 'Women' and (i_color = 'cream' or i_color = 'ghost') and (i_units = 'Ton' or i_units = 'Gross') and (i_size = 'economy' or i_size = 'small') ) or (i_category = 'Women' and (i_color = 'midnight' or i_color = 'burlywood') and (i_units = 'Tsp' or i_units = 'Bundle') and (i_size = 'medium' or i_size = 'extra large') ) or (i_category = 'Men' and (i_color = 'lavender' or i_color = 'azure') and (i_units = 'Each' or i_units = 'Lb') and (i_size = 'large' or i_size = 'N/A') ) or (i_category = 'Men' and (i_color = 'chocolate' or i_color = 'steel') and (i_units = 'N/A' or i_units = 'Dozen') and (i_size = 'economy' or i_size = 'small') ))) or (i_manufact = i1.i_manufact and ((i_category = 'Women' and (i_color = 'floral' or i_color = 'royal') and (i_units = 'Unknown' or i_units = 'Tbl') and (i_size = 'economy' or i_size = 'small') ) or (i_category = 'Women' and (i_color = 'navy' or i_color = 'forest') and (i_units = 'Bunch' or i_units = 'Dram') and (i_size = 'medium' or i_size = 'extra large') ) or (i_category = 'Men' and (i_color = 'cyan' or i_color = 'indian') and (i_units = 'Carton' or i_units = 'Cup') and (i_size = 'large' or i_size = 'N/A') ) or (i_category = 'Men' and (i_color = 'coral' or i_color = 'pale') and (i_units = 'Pallet' or i_units = 'Gram') and (i_size = 'economy' or i_size = 'small') )))) > 0 order by i_product_name LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q42.sql ================================================ -- CometBench-DS query 42 derived from TPC-DS query 42 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select dt.d_year ,item.i_category_id ,item.i_category ,sum(ss_ext_sales_price) from date_dim dt ,store_sales ,item where dt.d_date_sk = store_sales.ss_sold_date_sk and store_sales.ss_item_sk = item.i_item_sk and item.i_manager_id = 1 and dt.d_moy=11 and dt.d_year=1998 group by dt.d_year ,item.i_category_id ,item.i_category order by sum(ss_ext_sales_price) desc,dt.d_year ,item.i_category_id ,item.i_category LIMIT 100 ; ================================================ FILE: benchmarks/tpc/queries/tpcds/q43.sql ================================================ -- CometBench-DS query 43 derived from TPC-DS query 43 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select s_store_name, s_store_id, sum(case when (d_day_name='Sunday') then ss_sales_price else null end) sun_sales, sum(case when (d_day_name='Monday') then ss_sales_price else null end) mon_sales, sum(case when (d_day_name='Tuesday') then ss_sales_price else null end) tue_sales, sum(case when (d_day_name='Wednesday') then ss_sales_price else null end) wed_sales, sum(case when (d_day_name='Thursday') then ss_sales_price else null end) thu_sales, sum(case when (d_day_name='Friday') then ss_sales_price else null end) fri_sales, sum(case when (d_day_name='Saturday') then ss_sales_price else null end) sat_sales from date_dim, store_sales, store where d_date_sk = ss_sold_date_sk and s_store_sk = ss_store_sk and s_gmt_offset = -5 and d_year = 2000 group by s_store_name, s_store_id order by s_store_name, s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q44.sql ================================================ -- CometBench-DS query 44 derived from TPC-DS query 44 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select asceding.rnk, i1.i_product_name best_performing, i2.i_product_name worst_performing from(select * from (select item_sk,rank() over (order by rank_col asc) rnk from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col from store_sales ss1 where ss_store_sk = 6 group by ss_item_sk having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col from store_sales where ss_store_sk = 6 and ss_hdemo_sk is null group by ss_store_sk))V1)V11 where rnk < 11) asceding, (select * from (select item_sk,rank() over (order by rank_col desc) rnk from (select ss_item_sk item_sk,avg(ss_net_profit) rank_col from store_sales ss1 where ss_store_sk = 6 group by ss_item_sk having avg(ss_net_profit) > 0.9*(select avg(ss_net_profit) rank_col from store_sales where ss_store_sk = 6 and ss_hdemo_sk is null group by ss_store_sk))V2)V21 where rnk < 11) descending, item i1, item i2 where asceding.rnk = descending.rnk and i1.i_item_sk=asceding.item_sk and i2.i_item_sk=descending.item_sk order by asceding.rnk LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q45.sql ================================================ -- CometBench-DS query 45 derived from TPC-DS query 45 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select ca_zip, ca_city, sum(ws_sales_price) from web_sales, customer, customer_address, date_dim, item where ws_bill_customer_sk = c_customer_sk and c_current_addr_sk = ca_address_sk and ws_item_sk = i_item_sk and ( substr(ca_zip,1,5) in ('85669', '86197','88274','83405','86475', '85392', '85460', '80348', '81792') or i_item_id in (select i_item_id from item where i_item_sk in (2, 3, 5, 7, 11, 13, 17, 19, 23, 29) ) ) and ws_sold_date_sk = d_date_sk and d_qoy = 2 and d_year = 2000 group by ca_zip, ca_city order by ca_zip, ca_city LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q46.sql ================================================ -- CometBench-DS query 46 derived from TPC-DS query 46 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select c_last_name ,c_first_name ,ca_city ,bought_city ,ss_ticket_number ,amt,profit from (select ss_ticket_number ,ss_customer_sk ,ca_city bought_city ,sum(ss_coupon_amt) amt ,sum(ss_net_profit) profit from store_sales,date_dim,store,household_demographics,customer_address where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and store_sales.ss_addr_sk = customer_address.ca_address_sk and (household_demographics.hd_dep_count = 3 or household_demographics.hd_vehicle_count= 1) and date_dim.d_dow in (6,0) and date_dim.d_year in (1999,1999+1,1999+2) and store.s_city in ('Midway','Fairview','Fairview','Midway','Fairview') group by ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city) dn,customer,customer_address current_addr where ss_customer_sk = c_customer_sk and customer.c_current_addr_sk = current_addr.ca_address_sk and current_addr.ca_city <> bought_city order by c_last_name ,c_first_name ,ca_city ,bought_city ,ss_ticket_number LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q47.sql ================================================ -- CometBench-DS query 47 derived from TPC-DS query 47 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. with v1 as( select i_category, i_brand, s_store_name, s_company_name, d_year, d_moy, sum(ss_sales_price) sum_sales, avg(sum(ss_sales_price)) over (partition by i_category, i_brand, s_store_name, s_company_name, d_year) avg_monthly_sales, rank() over (partition by i_category, i_brand, s_store_name, s_company_name order by d_year, d_moy) rn from item, store_sales, date_dim, store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and ( d_year = 2001 or ( d_year = 2001-1 and d_moy =12) or ( d_year = 2001+1 and d_moy =1) ) group by i_category, i_brand, s_store_name, s_company_name, d_year, d_moy), v2 as( select v1.i_category, v1.i_brand, v1.s_store_name, v1.s_company_name ,v1.d_year ,v1.avg_monthly_sales ,v1.sum_sales, v1_lag.sum_sales psum, v1_lead.sum_sales nsum from v1, v1 v1_lag, v1 v1_lead where v1.i_category = v1_lag.i_category and v1.i_category = v1_lead.i_category and v1.i_brand = v1_lag.i_brand and v1.i_brand = v1_lead.i_brand and v1.s_store_name = v1_lag.s_store_name and v1.s_store_name = v1_lead.s_store_name and v1.s_company_name = v1_lag.s_company_name and v1.s_company_name = v1_lead.s_company_name and v1.rn = v1_lag.rn + 1 and v1.rn = v1_lead.rn - 1) select * from v2 where d_year = 2001 and avg_monthly_sales > 0 and case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 order by sum_sales - avg_monthly_sales, nsum LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q48.sql ================================================ -- CometBench-DS query 48 derived from TPC-DS query 48 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select sum (ss_quantity) from store_sales, store, customer_demographics, customer_address, date_dim where s_store_sk = ss_store_sk and ss_sold_date_sk = d_date_sk and d_year = 2001 and ( ( cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'W' and cd_education_status = '2 yr Degree' and ss_sales_price between 100.00 and 150.00 ) or ( cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'S' and cd_education_status = 'Advanced Degree' and ss_sales_price between 50.00 and 100.00 ) or ( cd_demo_sk = ss_cdemo_sk and cd_marital_status = 'D' and cd_education_status = 'Primary' and ss_sales_price between 150.00 and 200.00 ) ) and ( ( ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('IL', 'KY', 'OR') and ss_net_profit between 0 and 2000 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('VA', 'FL', 'AL') and ss_net_profit between 150 and 3000 ) or (ss_addr_sk = ca_address_sk and ca_country = 'United States' and ca_state in ('OK', 'IA', 'TX') and ss_net_profit between 50 and 25000 ) ) ; ================================================ FILE: benchmarks/tpc/queries/tpcds/q49.sql ================================================ -- CometBench-DS query 49 derived from TPC-DS query 49 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select channel, item, return_ratio, return_rank, currency_rank from (select 'web' as channel ,web.item ,web.return_ratio ,web.return_rank ,web.currency_rank from ( select item ,return_ratio ,currency_ratio ,rank() over (order by return_ratio) as return_rank ,rank() over (order by currency_ratio) as currency_rank from ( select ws.ws_item_sk as item ,(cast(sum(coalesce(wr.wr_return_quantity,0)) as decimal(15,4))/ cast(sum(coalesce(ws.ws_quantity,0)) as decimal(15,4) )) as return_ratio ,(cast(sum(coalesce(wr.wr_return_amt,0)) as decimal(15,4))/ cast(sum(coalesce(ws.ws_net_paid,0)) as decimal(15,4) )) as currency_ratio from web_sales ws left outer join web_returns wr on (ws.ws_order_number = wr.wr_order_number and ws.ws_item_sk = wr.wr_item_sk) ,date_dim where wr.wr_return_amt > 10000 and ws.ws_net_profit > 1 and ws.ws_net_paid > 0 and ws.ws_quantity > 0 and ws_sold_date_sk = d_date_sk and d_year = 2000 and d_moy = 12 group by ws.ws_item_sk ) in_web ) web where ( web.return_rank <= 10 or web.currency_rank <= 10 ) union select 'catalog' as channel ,catalog.item ,catalog.return_ratio ,catalog.return_rank ,catalog.currency_rank from ( select item ,return_ratio ,currency_ratio ,rank() over (order by return_ratio) as return_rank ,rank() over (order by currency_ratio) as currency_rank from ( select cs.cs_item_sk as item ,(cast(sum(coalesce(cr.cr_return_quantity,0)) as decimal(15,4))/ cast(sum(coalesce(cs.cs_quantity,0)) as decimal(15,4) )) as return_ratio ,(cast(sum(coalesce(cr.cr_return_amount,0)) as decimal(15,4))/ cast(sum(coalesce(cs.cs_net_paid,0)) as decimal(15,4) )) as currency_ratio from catalog_sales cs left outer join catalog_returns cr on (cs.cs_order_number = cr.cr_order_number and cs.cs_item_sk = cr.cr_item_sk) ,date_dim where cr.cr_return_amount > 10000 and cs.cs_net_profit > 1 and cs.cs_net_paid > 0 and cs.cs_quantity > 0 and cs_sold_date_sk = d_date_sk and d_year = 2000 and d_moy = 12 group by cs.cs_item_sk ) in_cat ) catalog where ( catalog.return_rank <= 10 or catalog.currency_rank <=10 ) union select 'store' as channel ,store.item ,store.return_ratio ,store.return_rank ,store.currency_rank from ( select item ,return_ratio ,currency_ratio ,rank() over (order by return_ratio) as return_rank ,rank() over (order by currency_ratio) as currency_rank from ( select sts.ss_item_sk as item ,(cast(sum(coalesce(sr.sr_return_quantity,0)) as decimal(15,4))/cast(sum(coalesce(sts.ss_quantity,0)) as decimal(15,4) )) as return_ratio ,(cast(sum(coalesce(sr.sr_return_amt,0)) as decimal(15,4))/cast(sum(coalesce(sts.ss_net_paid,0)) as decimal(15,4) )) as currency_ratio from store_sales sts left outer join store_returns sr on (sts.ss_ticket_number = sr.sr_ticket_number and sts.ss_item_sk = sr.sr_item_sk) ,date_dim where sr.sr_return_amt > 10000 and sts.ss_net_profit > 1 and sts.ss_net_paid > 0 and sts.ss_quantity > 0 and ss_sold_date_sk = d_date_sk and d_year = 2000 and d_moy = 12 group by sts.ss_item_sk ) in_store ) store where ( store.return_rank <= 10 or store.currency_rank <= 10 ) ) order by 1,4,5,2 LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q5.sql ================================================ -- CometBench-DS query 5 derived from TPC-DS query 5 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. with ssr as (select s_store_id, sum(sales_price) as sales, sum(profit) as profit, sum(return_amt) as returns, sum(net_loss) as profit_loss from ( select ss_store_sk as store_sk, ss_sold_date_sk as date_sk, ss_ext_sales_price as sales_price, ss_net_profit as profit, cast(0 as decimal(7,2)) as return_amt, cast(0 as decimal(7,2)) as net_loss from store_sales union all select sr_store_sk as store_sk, sr_returned_date_sk as date_sk, cast(0 as decimal(7,2)) as sales_price, cast(0 as decimal(7,2)) as profit, sr_return_amt as return_amt, sr_net_loss as net_loss from store_returns ) salesreturns, date_dim, store where date_sk = d_date_sk and d_date between cast('2001-08-04' as date) and (cast('2001-08-04' as date) + INTERVAL '14 DAYS') and store_sk = s_store_sk group by s_store_id) , csr as (select cp_catalog_page_id, sum(sales_price) as sales, sum(profit) as profit, sum(return_amt) as returns, sum(net_loss) as profit_loss from ( select cs_catalog_page_sk as page_sk, cs_sold_date_sk as date_sk, cs_ext_sales_price as sales_price, cs_net_profit as profit, cast(0 as decimal(7,2)) as return_amt, cast(0 as decimal(7,2)) as net_loss from catalog_sales union all select cr_catalog_page_sk as page_sk, cr_returned_date_sk as date_sk, cast(0 as decimal(7,2)) as sales_price, cast(0 as decimal(7,2)) as profit, cr_return_amount as return_amt, cr_net_loss as net_loss from catalog_returns ) salesreturns, date_dim, catalog_page where date_sk = d_date_sk and d_date between cast('2001-08-04' as date) and (cast('2001-08-04' as date) + INTERVAL '14 DAYS') and page_sk = cp_catalog_page_sk group by cp_catalog_page_id) , wsr as (select web_site_id, sum(sales_price) as sales, sum(profit) as profit, sum(return_amt) as returns, sum(net_loss) as profit_loss from ( select ws_web_site_sk as wsr_web_site_sk, ws_sold_date_sk as date_sk, ws_ext_sales_price as sales_price, ws_net_profit as profit, cast(0 as decimal(7,2)) as return_amt, cast(0 as decimal(7,2)) as net_loss from web_sales union all select ws_web_site_sk as wsr_web_site_sk, wr_returned_date_sk as date_sk, cast(0 as decimal(7,2)) as sales_price, cast(0 as decimal(7,2)) as profit, wr_return_amt as return_amt, wr_net_loss as net_loss from web_returns left outer join web_sales on ( wr_item_sk = ws_item_sk and wr_order_number = ws_order_number) ) salesreturns, date_dim, web_site where date_sk = d_date_sk and d_date between cast('2001-08-04' as date) and (cast('2001-08-04' as date) + INTERVAL '14 DAYS') and wsr_web_site_sk = web_site_sk group by web_site_id) select channel , id , sum(sales) as sales , sum(returns) as returns , sum(profit) as profit from (select 'store channel' as channel , 'store' || s_store_id as id , sales , returns , (profit - profit_loss) as profit from ssr union all select 'catalog channel' as channel , 'catalog_page' || cp_catalog_page_id as id , sales , returns , (profit - profit_loss) as profit from csr union all select 'web channel' as channel , 'web_site' || web_site_id as id , sales , returns , (profit - profit_loss) as profit from wsr ) x group by rollup (channel, id) order by channel ,id LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q50.sql ================================================ -- CometBench-DS query 50 derived from TPC-DS query 50 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select s_store_name ,s_company_id ,s_street_number ,s_street_name ,s_street_type ,s_suite_number ,s_city ,s_county ,s_state ,s_zip ,sum(case when (sr_returned_date_sk - ss_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 30) and (sr_returned_date_sk - ss_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 60) and (sr_returned_date_sk - ss_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 90) and (sr_returned_date_sk - ss_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` ,sum(case when (sr_returned_date_sk - ss_sold_date_sk > 120) then 1 else 0 end) as `>120 days` from store_sales ,store_returns ,store ,date_dim d1 ,date_dim d2 where d2.d_year = 2002 and d2.d_moy = 8 and ss_ticket_number = sr_ticket_number and ss_item_sk = sr_item_sk and ss_sold_date_sk = d1.d_date_sk and sr_returned_date_sk = d2.d_date_sk and ss_customer_sk = sr_customer_sk and ss_store_sk = s_store_sk group by s_store_name ,s_company_id ,s_street_number ,s_street_name ,s_street_type ,s_suite_number ,s_city ,s_county ,s_state ,s_zip order by s_store_name ,s_company_id ,s_street_number ,s_street_name ,s_street_type ,s_suite_number ,s_city ,s_county ,s_state ,s_zip LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q51.sql ================================================ -- CometBench-DS query 51 derived from TPC-DS query 51 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. WITH web_v1 as ( select ws_item_sk item_sk, d_date, sum(sum(ws_sales_price)) over (partition by ws_item_sk order by d_date rows between unbounded preceding and current row) cume_sales from web_sales ,date_dim where ws_sold_date_sk=d_date_sk and d_month_seq between 1215 and 1215+11 and ws_item_sk is not NULL group by ws_item_sk, d_date), store_v1 as ( select ss_item_sk item_sk, d_date, sum(sum(ss_sales_price)) over (partition by ss_item_sk order by d_date rows between unbounded preceding and current row) cume_sales from store_sales ,date_dim where ss_sold_date_sk=d_date_sk and d_month_seq between 1215 and 1215+11 and ss_item_sk is not NULL group by ss_item_sk, d_date) select * from (select item_sk ,d_date ,web_sales ,store_sales ,max(web_sales) over (partition by item_sk order by d_date rows between unbounded preceding and current row) web_cumulative ,max(store_sales) over (partition by item_sk order by d_date rows between unbounded preceding and current row) store_cumulative from (select case when web.item_sk is not null then web.item_sk else store.item_sk end item_sk ,case when web.d_date is not null then web.d_date else store.d_date end d_date ,web.cume_sales web_sales ,store.cume_sales store_sales from web_v1 web full outer join store_v1 store on (web.item_sk = store.item_sk and web.d_date = store.d_date) )x )y where web_cumulative > store_cumulative order by item_sk ,d_date LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q52.sql ================================================ -- CometBench-DS query 52 derived from TPC-DS query 52 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select dt.d_year ,item.i_brand_id brand_id ,item.i_brand brand ,sum(ss_ext_sales_price) ext_price from date_dim dt ,store_sales ,item where dt.d_date_sk = store_sales.ss_sold_date_sk and store_sales.ss_item_sk = item.i_item_sk and item.i_manager_id = 1 and dt.d_moy=11 and dt.d_year=2000 group by dt.d_year ,item.i_brand ,item.i_brand_id order by dt.d_year ,ext_price desc ,brand_id LIMIT 100 ; ================================================ FILE: benchmarks/tpc/queries/tpcds/q53.sql ================================================ -- CometBench-DS query 53 derived from TPC-DS query 53 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select * from (select i_manufact_id, sum(ss_sales_price) sum_sales, avg(sum(ss_sales_price)) over (partition by i_manufact_id) avg_quarterly_sales from item, store_sales, date_dim, store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and d_month_seq in (1197,1197+1,1197+2,1197+3,1197+4,1197+5,1197+6,1197+7,1197+8,1197+9,1197+10,1197+11) and ((i_category in ('Books','Children','Electronics') and i_class in ('personal','portable','reference','self-help') and i_brand in ('scholaramalgamalg #14','scholaramalgamalg #7', 'exportiunivamalg #9','scholaramalgamalg #9')) or(i_category in ('Women','Music','Men') and i_class in ('accessories','classical','fragrances','pants') and i_brand in ('amalgimporto #1','edu packscholar #1','exportiimporto #1', 'importoamalg #1'))) group by i_manufact_id, d_qoy ) tmp1 where case when avg_quarterly_sales > 0 then abs (sum_sales - avg_quarterly_sales)/ avg_quarterly_sales else null end > 0.1 order by avg_quarterly_sales, sum_sales, i_manufact_id LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q54.sql ================================================ -- CometBench-DS query 54 derived from TPC-DS query 54 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. with my_customers as ( select distinct c_customer_sk , c_current_addr_sk from ( select cs_sold_date_sk sold_date_sk, cs_bill_customer_sk customer_sk, cs_item_sk item_sk from catalog_sales union all select ws_sold_date_sk sold_date_sk, ws_bill_customer_sk customer_sk, ws_item_sk item_sk from web_sales ) cs_or_ws_sales, item, date_dim, customer where sold_date_sk = d_date_sk and item_sk = i_item_sk and i_category = 'Men' and i_class = 'shirts' and c_customer_sk = cs_or_ws_sales.customer_sk and d_moy = 4 and d_year = 1998 ) , my_revenue as ( select c_customer_sk, sum(ss_ext_sales_price) as revenue from my_customers, store_sales, customer_address, store, date_dim where c_current_addr_sk = ca_address_sk and ca_county = s_county and ca_state = s_state and ss_sold_date_sk = d_date_sk and c_customer_sk = ss_customer_sk and d_month_seq between (select distinct d_month_seq+1 from date_dim where d_year = 1998 and d_moy = 4) and (select distinct d_month_seq+3 from date_dim where d_year = 1998 and d_moy = 4) group by c_customer_sk ) , segments as (select cast((revenue/50) as int) as segment from my_revenue ) select segment, count(*) as num_customers, segment*50 as segment_base from segments group by segment order by segment, num_customers LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q55.sql ================================================ -- CometBench-DS query 55 derived from TPC-DS query 55 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select i_brand_id brand_id, i_brand brand, sum(ss_ext_sales_price) ext_price from date_dim, store_sales, item where d_date_sk = ss_sold_date_sk and ss_item_sk = i_item_sk and i_manager_id=20 and d_moy=12 and d_year=1998 group by i_brand, i_brand_id order by ext_price desc, i_brand_id LIMIT 100 ; ================================================ FILE: benchmarks/tpc/queries/tpcds/q56.sql ================================================ -- CometBench-DS query 56 derived from TPC-DS query 56 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. with ss as ( select i_item_id,sum(ss_ext_sales_price) total_sales from store_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_color in ('powder','goldenrod','bisque')) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 1998 and d_moy = 5 and ss_addr_sk = ca_address_sk and ca_gmt_offset = -5 group by i_item_id), cs as ( select i_item_id,sum(cs_ext_sales_price) total_sales from catalog_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_color in ('powder','goldenrod','bisque')) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 1998 and d_moy = 5 and cs_bill_addr_sk = ca_address_sk and ca_gmt_offset = -5 group by i_item_id), ws as ( select i_item_id,sum(ws_ext_sales_price) total_sales from web_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_color in ('powder','goldenrod','bisque')) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 1998 and d_moy = 5 and ws_bill_addr_sk = ca_address_sk and ca_gmt_offset = -5 group by i_item_id) select i_item_id ,sum(total_sales) total_sales from (select * from ss union all select * from cs union all select * from ws) tmp1 group by i_item_id order by total_sales, i_item_id LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q57.sql ================================================ -- CometBench-DS query 57 derived from TPC-DS query 57 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. with v1 as( select i_category, i_brand, cc_name, d_year, d_moy, sum(cs_sales_price) sum_sales, avg(sum(cs_sales_price)) over (partition by i_category, i_brand, cc_name, d_year) avg_monthly_sales, rank() over (partition by i_category, i_brand, cc_name order by d_year, d_moy) rn from item, catalog_sales, date_dim, call_center where cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and cc_call_center_sk= cs_call_center_sk and ( d_year = 2000 or ( d_year = 2000-1 and d_moy =12) or ( d_year = 2000+1 and d_moy =1) ) group by i_category, i_brand, cc_name , d_year, d_moy), v2 as( select v1.cc_name ,v1.d_year, v1.d_moy ,v1.avg_monthly_sales ,v1.sum_sales, v1_lag.sum_sales psum, v1_lead.sum_sales nsum from v1, v1 v1_lag, v1 v1_lead where v1.i_category = v1_lag.i_category and v1.i_category = v1_lead.i_category and v1.i_brand = v1_lag.i_brand and v1.i_brand = v1_lead.i_brand and v1. cc_name = v1_lag. cc_name and v1. cc_name = v1_lead. cc_name and v1.rn = v1_lag.rn + 1 and v1.rn = v1_lead.rn - 1) select * from v2 where d_year = 2000 and avg_monthly_sales > 0 and case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 order by sum_sales - avg_monthly_sales, psum LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q58.sql ================================================ -- CometBench-DS query 58 derived from TPC-DS query 58 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. with ss_items as (select i_item_id item_id ,sum(ss_ext_sales_price) ss_item_rev from store_sales ,item ,date_dim where ss_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq = (select d_week_seq from date_dim where d_date = '2000-02-12')) and ss_sold_date_sk = d_date_sk group by i_item_id), cs_items as (select i_item_id item_id ,sum(cs_ext_sales_price) cs_item_rev from catalog_sales ,item ,date_dim where cs_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq = (select d_week_seq from date_dim where d_date = '2000-02-12')) and cs_sold_date_sk = d_date_sk group by i_item_id), ws_items as (select i_item_id item_id ,sum(ws_ext_sales_price) ws_item_rev from web_sales ,item ,date_dim where ws_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq =(select d_week_seq from date_dim where d_date = '2000-02-12')) and ws_sold_date_sk = d_date_sk group by i_item_id) select ss_items.item_id ,ss_item_rev ,ss_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ss_dev ,cs_item_rev ,cs_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 cs_dev ,ws_item_rev ,ws_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ws_dev ,(ss_item_rev+cs_item_rev+ws_item_rev)/3 average from ss_items,cs_items,ws_items where ss_items.item_id=cs_items.item_id and ss_items.item_id=ws_items.item_id and ss_item_rev between 0.9 * cs_item_rev and 1.1 * cs_item_rev and ss_item_rev between 0.9 * ws_item_rev and 1.1 * ws_item_rev and cs_item_rev between 0.9 * ss_item_rev and 1.1 * ss_item_rev and cs_item_rev between 0.9 * ws_item_rev and 1.1 * ws_item_rev and ws_item_rev between 0.9 * ss_item_rev and 1.1 * ss_item_rev and ws_item_rev between 0.9 * cs_item_rev and 1.1 * cs_item_rev order by item_id ,ss_item_rev LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q59.sql ================================================ -- CometBench-DS query 59 derived from TPC-DS query 59 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. with wss as (select d_week_seq, ss_store_sk, sum(case when (d_day_name='Sunday') then ss_sales_price else null end) sun_sales, sum(case when (d_day_name='Monday') then ss_sales_price else null end) mon_sales, sum(case when (d_day_name='Tuesday') then ss_sales_price else null end) tue_sales, sum(case when (d_day_name='Wednesday') then ss_sales_price else null end) wed_sales, sum(case when (d_day_name='Thursday') then ss_sales_price else null end) thu_sales, sum(case when (d_day_name='Friday') then ss_sales_price else null end) fri_sales, sum(case when (d_day_name='Saturday') then ss_sales_price else null end) sat_sales from store_sales,date_dim where d_date_sk = ss_sold_date_sk group by d_week_seq,ss_store_sk ) select s_store_name1,s_store_id1,d_week_seq1 ,sun_sales1/sun_sales2,mon_sales1/mon_sales2 ,tue_sales1/tue_sales2,wed_sales1/wed_sales2,thu_sales1/thu_sales2 ,fri_sales1/fri_sales2,sat_sales1/sat_sales2 from (select s_store_name s_store_name1,wss.d_week_seq d_week_seq1 ,s_store_id s_store_id1,sun_sales sun_sales1 ,mon_sales mon_sales1,tue_sales tue_sales1 ,wed_sales wed_sales1,thu_sales thu_sales1 ,fri_sales fri_sales1,sat_sales sat_sales1 from wss,store,date_dim d where d.d_week_seq = wss.d_week_seq and ss_store_sk = s_store_sk and d_month_seq between 1206 and 1206 + 11) y, (select s_store_name s_store_name2,wss.d_week_seq d_week_seq2 ,s_store_id s_store_id2,sun_sales sun_sales2 ,mon_sales mon_sales2,tue_sales tue_sales2 ,wed_sales wed_sales2,thu_sales thu_sales2 ,fri_sales fri_sales2,sat_sales sat_sales2 from wss,store,date_dim d where d.d_week_seq = wss.d_week_seq and ss_store_sk = s_store_sk and d_month_seq between 1206+ 12 and 1206 + 23) x where s_store_id1=s_store_id2 and d_week_seq1=d_week_seq2-52 order by s_store_name1,s_store_id1,d_week_seq1 LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q6.sql ================================================ -- CometBench-DS query 6 derived from TPC-DS query 6 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select a.ca_state state, count(*) cnt from customer_address a ,customer c ,store_sales s ,date_dim d ,item i where a.ca_address_sk = c.c_current_addr_sk and c.c_customer_sk = s.ss_customer_sk and s.ss_sold_date_sk = d.d_date_sk and s.ss_item_sk = i.i_item_sk and d.d_month_seq = (select distinct (d_month_seq) from date_dim where d_year = 1998 and d_moy = 3 ) and i.i_current_price > 1.2 * (select avg(j.i_current_price) from item j where j.i_category = i.i_category) group by a.ca_state having count(*) >= 10 order by cnt, a.ca_state LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q60.sql ================================================ -- CometBench-DS query 60 derived from TPC-DS query 60 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. with ss as ( select i_item_id,sum(ss_ext_sales_price) total_sales from store_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_category in ('Shoes')) and ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and d_year = 2001 and d_moy = 10 and ss_addr_sk = ca_address_sk and ca_gmt_offset = -6 group by i_item_id), cs as ( select i_item_id,sum(cs_ext_sales_price) total_sales from catalog_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_category in ('Shoes')) and cs_item_sk = i_item_sk and cs_sold_date_sk = d_date_sk and d_year = 2001 and d_moy = 10 and cs_bill_addr_sk = ca_address_sk and ca_gmt_offset = -6 group by i_item_id), ws as ( select i_item_id,sum(ws_ext_sales_price) total_sales from web_sales, date_dim, customer_address, item where i_item_id in (select i_item_id from item where i_category in ('Shoes')) and ws_item_sk = i_item_sk and ws_sold_date_sk = d_date_sk and d_year = 2001 and d_moy = 10 and ws_bill_addr_sk = ca_address_sk and ca_gmt_offset = -6 group by i_item_id) select i_item_id ,sum(total_sales) total_sales from (select * from ss union all select * from cs union all select * from ws) tmp1 group by i_item_id order by i_item_id ,total_sales LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q61.sql ================================================ -- CometBench-DS query 61 derived from TPC-DS query 61 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select promotions,total,cast(promotions as decimal(15,4))/cast(total as decimal(15,4))*100 from (select sum(ss_ext_sales_price) promotions from store_sales ,store ,promotion ,date_dim ,customer ,customer_address ,item where ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and ss_promo_sk = p_promo_sk and ss_customer_sk= c_customer_sk and ca_address_sk = c_current_addr_sk and ss_item_sk = i_item_sk and ca_gmt_offset = -6 and i_category = 'Sports' and (p_channel_dmail = 'Y' or p_channel_email = 'Y' or p_channel_tv = 'Y') and s_gmt_offset = -6 and d_year = 2002 and d_moy = 11) promotional_sales, (select sum(ss_ext_sales_price) total from store_sales ,store ,date_dim ,customer ,customer_address ,item where ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and ss_customer_sk= c_customer_sk and ca_address_sk = c_current_addr_sk and ss_item_sk = i_item_sk and ca_gmt_offset = -6 and i_category = 'Sports' and s_gmt_offset = -6 and d_year = 2002 and d_moy = 11) all_sales order by promotions, total LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q62.sql ================================================ -- CometBench-DS query 62 derived from TPC-DS query 62 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select substr(w_warehouse_name,1,20) ,sm_type ,web_name ,sum(case when (ws_ship_date_sk - ws_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 30) and (ws_ship_date_sk - ws_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 60) and (ws_ship_date_sk - ws_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 90) and (ws_ship_date_sk - ws_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` ,sum(case when (ws_ship_date_sk - ws_sold_date_sk > 120) then 1 else 0 end) as `>120 days` from web_sales ,warehouse ,ship_mode ,web_site ,date_dim where d_month_seq between 1217 and 1217 + 11 and ws_ship_date_sk = d_date_sk and ws_warehouse_sk = w_warehouse_sk and ws_ship_mode_sk = sm_ship_mode_sk and ws_web_site_sk = web_site_sk group by substr(w_warehouse_name,1,20) ,sm_type ,web_name order by substr(w_warehouse_name,1,20) ,sm_type ,web_name LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q63.sql ================================================ -- CometBench-DS query 63 derived from TPC-DS query 63 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select * from (select i_manager_id ,sum(ss_sales_price) sum_sales ,avg(sum(ss_sales_price)) over (partition by i_manager_id) avg_monthly_sales from item ,store_sales ,date_dim ,store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and d_month_seq in (1181,1181+1,1181+2,1181+3,1181+4,1181+5,1181+6,1181+7,1181+8,1181+9,1181+10,1181+11) and (( i_category in ('Books','Children','Electronics') and i_class in ('personal','portable','reference','self-help') and i_brand in ('scholaramalgamalg #14','scholaramalgamalg #7', 'exportiunivamalg #9','scholaramalgamalg #9')) or( i_category in ('Women','Music','Men') and i_class in ('accessories','classical','fragrances','pants') and i_brand in ('amalgimporto #1','edu packscholar #1','exportiimporto #1', 'importoamalg #1'))) group by i_manager_id, d_moy) tmp1 where case when avg_monthly_sales > 0 then abs (sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1 order by i_manager_id ,avg_monthly_sales ,sum_sales LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q64.sql ================================================ -- CometBench-DS query 64 derived from TPC-DS query 64 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. with cs_ui as (select cs_item_sk ,sum(cs_ext_list_price) as sale,sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit) as refund from catalog_sales ,catalog_returns where cs_item_sk = cr_item_sk and cs_order_number = cr_order_number group by cs_item_sk having sum(cs_ext_list_price)>2*sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit)), cross_sales as (select i_product_name product_name ,i_item_sk item_sk ,s_store_name store_name ,s_zip store_zip ,ad1.ca_street_number b_street_number ,ad1.ca_street_name b_street_name ,ad1.ca_city b_city ,ad1.ca_zip b_zip ,ad2.ca_street_number c_street_number ,ad2.ca_street_name c_street_name ,ad2.ca_city c_city ,ad2.ca_zip c_zip ,d1.d_year as syear ,d2.d_year as fsyear ,d3.d_year s2year ,count(*) cnt ,sum(ss_wholesale_cost) s1 ,sum(ss_list_price) s2 ,sum(ss_coupon_amt) s3 FROM store_sales ,store_returns ,cs_ui ,date_dim d1 ,date_dim d2 ,date_dim d3 ,store ,customer ,customer_demographics cd1 ,customer_demographics cd2 ,promotion ,household_demographics hd1 ,household_demographics hd2 ,customer_address ad1 ,customer_address ad2 ,income_band ib1 ,income_band ib2 ,item WHERE ss_store_sk = s_store_sk AND ss_sold_date_sk = d1.d_date_sk AND ss_customer_sk = c_customer_sk AND ss_cdemo_sk= cd1.cd_demo_sk AND ss_hdemo_sk = hd1.hd_demo_sk AND ss_addr_sk = ad1.ca_address_sk and ss_item_sk = i_item_sk and ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and ss_item_sk = cs_ui.cs_item_sk and c_current_cdemo_sk = cd2.cd_demo_sk AND c_current_hdemo_sk = hd2.hd_demo_sk AND c_current_addr_sk = ad2.ca_address_sk and c_first_sales_date_sk = d2.d_date_sk and c_first_shipto_date_sk = d3.d_date_sk and ss_promo_sk = p_promo_sk and hd1.hd_income_band_sk = ib1.ib_income_band_sk and hd2.hd_income_band_sk = ib2.ib_income_band_sk and cd1.cd_marital_status <> cd2.cd_marital_status and i_color in ('light','cyan','burnished','green','almond','smoke') and i_current_price between 22 and 22 + 10 and i_current_price between 22 + 1 and 22 + 15 group by i_product_name ,i_item_sk ,s_store_name ,s_zip ,ad1.ca_street_number ,ad1.ca_street_name ,ad1.ca_city ,ad1.ca_zip ,ad2.ca_street_number ,ad2.ca_street_name ,ad2.ca_city ,ad2.ca_zip ,d1.d_year ,d2.d_year ,d3.d_year ) select cs1.product_name ,cs1.store_name ,cs1.store_zip ,cs1.b_street_number ,cs1.b_street_name ,cs1.b_city ,cs1.b_zip ,cs1.c_street_number ,cs1.c_street_name ,cs1.c_city ,cs1.c_zip ,cs1.syear ,cs1.cnt ,cs1.s1 as s11 ,cs1.s2 as s21 ,cs1.s3 as s31 ,cs2.s1 as s12 ,cs2.s2 as s22 ,cs2.s3 as s32 ,cs2.syear ,cs2.cnt from cross_sales cs1,cross_sales cs2 where cs1.item_sk=cs2.item_sk and cs1.syear = 2001 and cs2.syear = 2001 + 1 and cs2.cnt <= cs1.cnt and cs1.store_name = cs2.store_name and cs1.store_zip = cs2.store_zip order by cs1.product_name ,cs1.store_name ,cs2.cnt ,cs1.s1 ,cs2.s1; ================================================ FILE: benchmarks/tpc/queries/tpcds/q65.sql ================================================ -- CometBench-DS query 65 derived from TPC-DS query 65 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select s_store_name, i_item_desc, sc.revenue, i_current_price, i_wholesale_cost, i_brand from store, item, (select ss_store_sk, avg(revenue) as ave from (select ss_store_sk, ss_item_sk, sum(ss_sales_price) as revenue from store_sales, date_dim where ss_sold_date_sk = d_date_sk and d_month_seq between 1186 and 1186+11 group by ss_store_sk, ss_item_sk) sa group by ss_store_sk) sb, (select ss_store_sk, ss_item_sk, sum(ss_sales_price) as revenue from store_sales, date_dim where ss_sold_date_sk = d_date_sk and d_month_seq between 1186 and 1186+11 group by ss_store_sk, ss_item_sk) sc where sb.ss_store_sk = sc.ss_store_sk and sc.revenue <= 0.1 * sb.ave and s_store_sk = sc.ss_store_sk and i_item_sk = sc.ss_item_sk order by s_store_name, i_item_desc LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q66.sql ================================================ -- CometBench-DS query 66 derived from TPC-DS query 66 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,ship_carriers ,year ,sum(jan_sales) as jan_sales ,sum(feb_sales) as feb_sales ,sum(mar_sales) as mar_sales ,sum(apr_sales) as apr_sales ,sum(may_sales) as may_sales ,sum(jun_sales) as jun_sales ,sum(jul_sales) as jul_sales ,sum(aug_sales) as aug_sales ,sum(sep_sales) as sep_sales ,sum(oct_sales) as oct_sales ,sum(nov_sales) as nov_sales ,sum(dec_sales) as dec_sales ,sum(jan_sales/w_warehouse_sq_ft) as jan_sales_per_sq_foot ,sum(feb_sales/w_warehouse_sq_ft) as feb_sales_per_sq_foot ,sum(mar_sales/w_warehouse_sq_ft) as mar_sales_per_sq_foot ,sum(apr_sales/w_warehouse_sq_ft) as apr_sales_per_sq_foot ,sum(may_sales/w_warehouse_sq_ft) as may_sales_per_sq_foot ,sum(jun_sales/w_warehouse_sq_ft) as jun_sales_per_sq_foot ,sum(jul_sales/w_warehouse_sq_ft) as jul_sales_per_sq_foot ,sum(aug_sales/w_warehouse_sq_ft) as aug_sales_per_sq_foot ,sum(sep_sales/w_warehouse_sq_ft) as sep_sales_per_sq_foot ,sum(oct_sales/w_warehouse_sq_ft) as oct_sales_per_sq_foot ,sum(nov_sales/w_warehouse_sq_ft) as nov_sales_per_sq_foot ,sum(dec_sales/w_warehouse_sq_ft) as dec_sales_per_sq_foot ,sum(jan_net) as jan_net ,sum(feb_net) as feb_net ,sum(mar_net) as mar_net ,sum(apr_net) as apr_net ,sum(may_net) as may_net ,sum(jun_net) as jun_net ,sum(jul_net) as jul_net ,sum(aug_net) as aug_net ,sum(sep_net) as sep_net ,sum(oct_net) as oct_net ,sum(nov_net) as nov_net ,sum(dec_net) as dec_net from ( select w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,'FEDEX' || ',' || 'GERMA' as ship_carriers ,d_year as year ,sum(case when d_moy = 1 then ws_ext_list_price* ws_quantity else 0 end) as jan_sales ,sum(case when d_moy = 2 then ws_ext_list_price* ws_quantity else 0 end) as feb_sales ,sum(case when d_moy = 3 then ws_ext_list_price* ws_quantity else 0 end) as mar_sales ,sum(case when d_moy = 4 then ws_ext_list_price* ws_quantity else 0 end) as apr_sales ,sum(case when d_moy = 5 then ws_ext_list_price* ws_quantity else 0 end) as may_sales ,sum(case when d_moy = 6 then ws_ext_list_price* ws_quantity else 0 end) as jun_sales ,sum(case when d_moy = 7 then ws_ext_list_price* ws_quantity else 0 end) as jul_sales ,sum(case when d_moy = 8 then ws_ext_list_price* ws_quantity else 0 end) as aug_sales ,sum(case when d_moy = 9 then ws_ext_list_price* ws_quantity else 0 end) as sep_sales ,sum(case when d_moy = 10 then ws_ext_list_price* ws_quantity else 0 end) as oct_sales ,sum(case when d_moy = 11 then ws_ext_list_price* ws_quantity else 0 end) as nov_sales ,sum(case when d_moy = 12 then ws_ext_list_price* ws_quantity else 0 end) as dec_sales ,sum(case when d_moy = 1 then ws_net_profit * ws_quantity else 0 end) as jan_net ,sum(case when d_moy = 2 then ws_net_profit * ws_quantity else 0 end) as feb_net ,sum(case when d_moy = 3 then ws_net_profit * ws_quantity else 0 end) as mar_net ,sum(case when d_moy = 4 then ws_net_profit * ws_quantity else 0 end) as apr_net ,sum(case when d_moy = 5 then ws_net_profit * ws_quantity else 0 end) as may_net ,sum(case when d_moy = 6 then ws_net_profit * ws_quantity else 0 end) as jun_net ,sum(case when d_moy = 7 then ws_net_profit * ws_quantity else 0 end) as jul_net ,sum(case when d_moy = 8 then ws_net_profit * ws_quantity else 0 end) as aug_net ,sum(case when d_moy = 9 then ws_net_profit * ws_quantity else 0 end) as sep_net ,sum(case when d_moy = 10 then ws_net_profit * ws_quantity else 0 end) as oct_net ,sum(case when d_moy = 11 then ws_net_profit * ws_quantity else 0 end) as nov_net ,sum(case when d_moy = 12 then ws_net_profit * ws_quantity else 0 end) as dec_net from web_sales ,warehouse ,date_dim ,time_dim ,ship_mode where ws_warehouse_sk = w_warehouse_sk and ws_sold_date_sk = d_date_sk and ws_sold_time_sk = t_time_sk and ws_ship_mode_sk = sm_ship_mode_sk and d_year = 2001 and t_time between 19072 and 19072+28800 and sm_carrier in ('FEDEX','GERMA') group by w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,d_year union all select w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,'FEDEX' || ',' || 'GERMA' as ship_carriers ,d_year as year ,sum(case when d_moy = 1 then cs_sales_price* cs_quantity else 0 end) as jan_sales ,sum(case when d_moy = 2 then cs_sales_price* cs_quantity else 0 end) as feb_sales ,sum(case when d_moy = 3 then cs_sales_price* cs_quantity else 0 end) as mar_sales ,sum(case when d_moy = 4 then cs_sales_price* cs_quantity else 0 end) as apr_sales ,sum(case when d_moy = 5 then cs_sales_price* cs_quantity else 0 end) as may_sales ,sum(case when d_moy = 6 then cs_sales_price* cs_quantity else 0 end) as jun_sales ,sum(case when d_moy = 7 then cs_sales_price* cs_quantity else 0 end) as jul_sales ,sum(case when d_moy = 8 then cs_sales_price* cs_quantity else 0 end) as aug_sales ,sum(case when d_moy = 9 then cs_sales_price* cs_quantity else 0 end) as sep_sales ,sum(case when d_moy = 10 then cs_sales_price* cs_quantity else 0 end) as oct_sales ,sum(case when d_moy = 11 then cs_sales_price* cs_quantity else 0 end) as nov_sales ,sum(case when d_moy = 12 then cs_sales_price* cs_quantity else 0 end) as dec_sales ,sum(case when d_moy = 1 then cs_net_paid * cs_quantity else 0 end) as jan_net ,sum(case when d_moy = 2 then cs_net_paid * cs_quantity else 0 end) as feb_net ,sum(case when d_moy = 3 then cs_net_paid * cs_quantity else 0 end) as mar_net ,sum(case when d_moy = 4 then cs_net_paid * cs_quantity else 0 end) as apr_net ,sum(case when d_moy = 5 then cs_net_paid * cs_quantity else 0 end) as may_net ,sum(case when d_moy = 6 then cs_net_paid * cs_quantity else 0 end) as jun_net ,sum(case when d_moy = 7 then cs_net_paid * cs_quantity else 0 end) as jul_net ,sum(case when d_moy = 8 then cs_net_paid * cs_quantity else 0 end) as aug_net ,sum(case when d_moy = 9 then cs_net_paid * cs_quantity else 0 end) as sep_net ,sum(case when d_moy = 10 then cs_net_paid * cs_quantity else 0 end) as oct_net ,sum(case when d_moy = 11 then cs_net_paid * cs_quantity else 0 end) as nov_net ,sum(case when d_moy = 12 then cs_net_paid * cs_quantity else 0 end) as dec_net from catalog_sales ,warehouse ,date_dim ,time_dim ,ship_mode where cs_warehouse_sk = w_warehouse_sk and cs_sold_date_sk = d_date_sk and cs_sold_time_sk = t_time_sk and cs_ship_mode_sk = sm_ship_mode_sk and d_year = 2001 and t_time between 19072 AND 19072+28800 and sm_carrier in ('FEDEX','GERMA') group by w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,d_year ) x group by w_warehouse_name ,w_warehouse_sq_ft ,w_city ,w_county ,w_state ,w_country ,ship_carriers ,year order by w_warehouse_name LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q67.sql ================================================ -- CometBench-DS query 67 derived from TPC-DS query 67 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select * from (select i_category ,i_class ,i_brand ,i_product_name ,d_year ,d_qoy ,d_moy ,s_store_id ,sumsales ,rank() over (partition by i_category order by sumsales desc) rk from (select i_category ,i_class ,i_brand ,i_product_name ,d_year ,d_qoy ,d_moy ,s_store_id ,sum(coalesce(ss_sales_price*ss_quantity,0)) sumsales from store_sales ,date_dim ,store ,item where ss_sold_date_sk=d_date_sk and ss_item_sk=i_item_sk and ss_store_sk = s_store_sk and d_month_seq between 1194 and 1194+11 group by rollup(i_category, i_class, i_brand, i_product_name, d_year, d_qoy, d_moy,s_store_id))dw1) dw2 where rk <= 100 order by i_category ,i_class ,i_brand ,i_product_name ,d_year ,d_qoy ,d_moy ,s_store_id ,sumsales ,rk LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q68.sql ================================================ -- CometBench-DS query 68 derived from TPC-DS query 68 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select c_last_name ,c_first_name ,ca_city ,bought_city ,ss_ticket_number ,extended_price ,extended_tax ,list_price from (select ss_ticket_number ,ss_customer_sk ,ca_city bought_city ,sum(ss_ext_sales_price) extended_price ,sum(ss_ext_list_price) list_price ,sum(ss_ext_tax) extended_tax from store_sales ,date_dim ,store ,household_demographics ,customer_address where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and store_sales.ss_addr_sk = customer_address.ca_address_sk and date_dim.d_dom between 1 and 2 and (household_demographics.hd_dep_count = 8 or household_demographics.hd_vehicle_count= 3) and date_dim.d_year in (2000,2000+1,2000+2) and store.s_city in ('Midway','Fairview') group by ss_ticket_number ,ss_customer_sk ,ss_addr_sk,ca_city) dn ,customer ,customer_address current_addr where ss_customer_sk = c_customer_sk and customer.c_current_addr_sk = current_addr.ca_address_sk and current_addr.ca_city <> bought_city order by c_last_name ,ss_ticket_number LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q69.sql ================================================ -- CometBench-DS query 69 derived from TPC-DS query 69 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select cd_gender, cd_marital_status, cd_education_status, count(*) cnt1, cd_purchase_estimate, count(*) cnt2, cd_credit_rating, count(*) cnt3 from customer c,customer_address ca,customer_demographics where c.c_current_addr_sk = ca.ca_address_sk and ca_state in ('IN','VA','MS') and cd_demo_sk = c.c_current_cdemo_sk and exists (select * from store_sales,date_dim where c.c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year = 2002 and d_moy between 2 and 2+2) and (not exists (select * from web_sales,date_dim where c.c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year = 2002 and d_moy between 2 and 2+2) and not exists (select * from catalog_sales,date_dim where c.c_customer_sk = cs_ship_customer_sk and cs_sold_date_sk = d_date_sk and d_year = 2002 and d_moy between 2 and 2+2)) group by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating order by cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q7.sql ================================================ -- CometBench-DS query 7 derived from TPC-DS query 7 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select i_item_id, avg(ss_quantity) agg1, avg(ss_list_price) agg2, avg(ss_coupon_amt) agg3, avg(ss_sales_price) agg4 from store_sales, customer_demographics, date_dim, item, promotion where ss_sold_date_sk = d_date_sk and ss_item_sk = i_item_sk and ss_cdemo_sk = cd_demo_sk and ss_promo_sk = p_promo_sk and cd_gender = 'M' and cd_marital_status = 'M' and cd_education_status = '4 yr Degree' and (p_channel_email = 'N' or p_channel_event = 'N') and d_year = 2001 group by i_item_id order by i_item_id LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q70.sql ================================================ -- CometBench-DS query 70 derived from TPC-DS query 70 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select sum(ss_net_profit) as total_sum ,s_state ,s_county ,grouping(s_state)+grouping(s_county) as lochierarchy ,rank() over ( partition by grouping(s_state)+grouping(s_county), case when grouping(s_county) = 0 then s_state end order by sum(ss_net_profit) desc) as rank_within_parent from store_sales ,date_dim d1 ,store where d1.d_month_seq between 1180 and 1180+11 and d1.d_date_sk = ss_sold_date_sk and s_store_sk = ss_store_sk and s_state in ( select s_state from (select s_state as s_state, rank() over ( partition by s_state order by sum(ss_net_profit) desc) as ranking from store_sales, store, date_dim where d_month_seq between 1180 and 1180+11 and d_date_sk = ss_sold_date_sk and s_store_sk = ss_store_sk group by s_state ) tmp1 where ranking <= 5 ) group by rollup(s_state,s_county) order by lochierarchy desc ,case when lochierarchy = 0 then s_state end ,rank_within_parent LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q71.sql ================================================ -- CometBench-DS query 71 derived from TPC-DS query 71 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select i_brand_id brand_id, i_brand brand,t_hour,t_minute, sum(ext_price) ext_price from item, (select ws_ext_sales_price as ext_price, ws_sold_date_sk as sold_date_sk, ws_item_sk as sold_item_sk, ws_sold_time_sk as time_sk from web_sales,date_dim where d_date_sk = ws_sold_date_sk and d_moy=11 and d_year=2001 union all select cs_ext_sales_price as ext_price, cs_sold_date_sk as sold_date_sk, cs_item_sk as sold_item_sk, cs_sold_time_sk as time_sk from catalog_sales,date_dim where d_date_sk = cs_sold_date_sk and d_moy=11 and d_year=2001 union all select ss_ext_sales_price as ext_price, ss_sold_date_sk as sold_date_sk, ss_item_sk as sold_item_sk, ss_sold_time_sk as time_sk from store_sales,date_dim where d_date_sk = ss_sold_date_sk and d_moy=11 and d_year=2001 ) tmp,time_dim where sold_item_sk = i_item_sk and i_manager_id=1 and time_sk = t_time_sk and (t_meal_time = 'breakfast' or t_meal_time = 'dinner') group by i_brand, i_brand_id,t_hour,t_minute order by ext_price desc, i_brand_id ; ================================================ FILE: benchmarks/tpc/queries/tpcds/q72.sql ================================================ -- CometBench-DS query 72 derived from TPC-DS query 72 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select i_item_desc ,w_warehouse_name ,d1.d_week_seq ,sum(case when p_promo_sk is null then 1 else 0 end) no_promo ,sum(case when p_promo_sk is not null then 1 else 0 end) promo ,count(*) total_cnt from catalog_sales join inventory on (cs_item_sk = inv_item_sk) join warehouse on (w_warehouse_sk=inv_warehouse_sk) join item on (i_item_sk = cs_item_sk) join customer_demographics on (cs_bill_cdemo_sk = cd_demo_sk) join household_demographics on (cs_bill_hdemo_sk = hd_demo_sk) join date_dim d1 on (cs_sold_date_sk = d1.d_date_sk) join date_dim d2 on (inv_date_sk = d2.d_date_sk) join date_dim d3 on (cs_ship_date_sk = d3.d_date_sk) left outer join promotion on (cs_promo_sk=p_promo_sk) left outer join catalog_returns on (cr_item_sk = cs_item_sk and cr_order_number = cs_order_number) where d1.d_week_seq = d2.d_week_seq and inv_quantity_on_hand < cs_quantity and d3.d_date > d1.d_date + 5 and hd_buy_potential = '501-1000' and d1.d_year = 1999 and cd_marital_status = 'S' group by i_item_desc,w_warehouse_name,d1.d_week_seq order by total_cnt desc, i_item_desc, w_warehouse_name, d_week_seq LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q73.sql ================================================ -- CometBench-DS query 73 derived from TPC-DS query 73 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select c_last_name ,c_first_name ,c_salutation ,c_preferred_cust_flag ,ss_ticket_number ,cnt from (select ss_ticket_number ,ss_customer_sk ,count(*) cnt from store_sales,date_dim,store,household_demographics where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and date_dim.d_dom between 1 and 2 and (household_demographics.hd_buy_potential = '1001-5000' or household_demographics.hd_buy_potential = '5001-10000') and household_demographics.hd_vehicle_count > 0 and case when household_demographics.hd_vehicle_count > 0 then household_demographics.hd_dep_count/ household_demographics.hd_vehicle_count else null end > 1 and date_dim.d_year in (1999,1999+1,1999+2) and store.s_county in ('Williamson County','Williamson County','Williamson County','Williamson County') group by ss_ticket_number,ss_customer_sk) dj,customer where ss_customer_sk = c_customer_sk and cnt between 1 and 5 order by cnt desc, c_last_name asc; ================================================ FILE: benchmarks/tpc/queries/tpcds/q74.sql ================================================ -- CometBench-DS query 74 derived from TPC-DS query 74 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. with year_total as ( select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,d_year as year ,stddev_samp(ss_net_paid) year_total ,'s' sale_type from customer ,store_sales ,date_dim where c_customer_sk = ss_customer_sk and ss_sold_date_sk = d_date_sk and d_year in (2001,2001+1) group by c_customer_id ,c_first_name ,c_last_name ,d_year union all select c_customer_id customer_id ,c_first_name customer_first_name ,c_last_name customer_last_name ,d_year as year ,stddev_samp(ws_net_paid) year_total ,'w' sale_type from customer ,web_sales ,date_dim where c_customer_sk = ws_bill_customer_sk and ws_sold_date_sk = d_date_sk and d_year in (2001,2001+1) group by c_customer_id ,c_first_name ,c_last_name ,d_year ) select t_s_secyear.customer_id, t_s_secyear.customer_first_name, t_s_secyear.customer_last_name from year_total t_s_firstyear ,year_total t_s_secyear ,year_total t_w_firstyear ,year_total t_w_secyear where t_s_secyear.customer_id = t_s_firstyear.customer_id and t_s_firstyear.customer_id = t_w_secyear.customer_id and t_s_firstyear.customer_id = t_w_firstyear.customer_id and t_s_firstyear.sale_type = 's' and t_w_firstyear.sale_type = 'w' and t_s_secyear.sale_type = 's' and t_w_secyear.sale_type = 'w' and t_s_firstyear.year = 2001 and t_s_secyear.year = 2001+1 and t_w_firstyear.year = 2001 and t_w_secyear.year = 2001+1 and t_s_firstyear.year_total > 0 and t_w_firstyear.year_total > 0 and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end order by 3,2,1 LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q75.sql ================================================ -- CometBench-DS query 75 derived from TPC-DS query 75 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. WITH all_sales AS ( SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,SUM(sales_cnt) AS sales_cnt ,SUM(sales_amt) AS sales_amt FROM (SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,cs_quantity - COALESCE(cr_return_quantity,0) AS sales_cnt ,cs_ext_sales_price - COALESCE(cr_return_amount,0.0) AS sales_amt FROM catalog_sales JOIN item ON i_item_sk=cs_item_sk JOIN date_dim ON d_date_sk=cs_sold_date_sk LEFT JOIN catalog_returns ON (cs_order_number=cr_order_number AND cs_item_sk=cr_item_sk) WHERE i_category='Shoes' UNION SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,ss_quantity - COALESCE(sr_return_quantity,0) AS sales_cnt ,ss_ext_sales_price - COALESCE(sr_return_amt,0.0) AS sales_amt FROM store_sales JOIN item ON i_item_sk=ss_item_sk JOIN date_dim ON d_date_sk=ss_sold_date_sk LEFT JOIN store_returns ON (ss_ticket_number=sr_ticket_number AND ss_item_sk=sr_item_sk) WHERE i_category='Shoes' UNION SELECT d_year ,i_brand_id ,i_class_id ,i_category_id ,i_manufact_id ,ws_quantity - COALESCE(wr_return_quantity,0) AS sales_cnt ,ws_ext_sales_price - COALESCE(wr_return_amt,0.0) AS sales_amt FROM web_sales JOIN item ON i_item_sk=ws_item_sk JOIN date_dim ON d_date_sk=ws_sold_date_sk LEFT JOIN web_returns ON (ws_order_number=wr_order_number AND ws_item_sk=wr_item_sk) WHERE i_category='Shoes') sales_detail GROUP BY d_year, i_brand_id, i_class_id, i_category_id, i_manufact_id) SELECT prev_yr.d_year AS prev_year ,curr_yr.d_year AS year ,curr_yr.i_brand_id ,curr_yr.i_class_id ,curr_yr.i_category_id ,curr_yr.i_manufact_id ,prev_yr.sales_cnt AS prev_yr_cnt ,curr_yr.sales_cnt AS curr_yr_cnt ,curr_yr.sales_cnt-prev_yr.sales_cnt AS sales_cnt_diff ,curr_yr.sales_amt-prev_yr.sales_amt AS sales_amt_diff FROM all_sales curr_yr, all_sales prev_yr WHERE curr_yr.i_brand_id=prev_yr.i_brand_id AND curr_yr.i_class_id=prev_yr.i_class_id AND curr_yr.i_category_id=prev_yr.i_category_id AND curr_yr.i_manufact_id=prev_yr.i_manufact_id AND curr_yr.d_year=2000 AND prev_yr.d_year=2000-1 AND CAST(curr_yr.sales_cnt AS DECIMAL(17,2))/CAST(prev_yr.sales_cnt AS DECIMAL(17,2))<0.9 ORDER BY sales_cnt_diff,sales_amt_diff LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q76.sql ================================================ -- CometBench-DS query 76 derived from TPC-DS query 76 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select channel, col_name, d_year, d_qoy, i_category, COUNT(*) sales_cnt, SUM(ext_sales_price) sales_amt FROM ( SELECT 'store' as channel, 'ss_customer_sk' col_name, d_year, d_qoy, i_category, ss_ext_sales_price ext_sales_price FROM store_sales, item, date_dim WHERE ss_customer_sk IS NULL AND ss_sold_date_sk=d_date_sk AND ss_item_sk=i_item_sk UNION ALL SELECT 'web' as channel, 'ws_ship_hdemo_sk' col_name, d_year, d_qoy, i_category, ws_ext_sales_price ext_sales_price FROM web_sales, item, date_dim WHERE ws_ship_hdemo_sk IS NULL AND ws_sold_date_sk=d_date_sk AND ws_item_sk=i_item_sk UNION ALL SELECT 'catalog' as channel, 'cs_bill_customer_sk' col_name, d_year, d_qoy, i_category, cs_ext_sales_price ext_sales_price FROM catalog_sales, item, date_dim WHERE cs_bill_customer_sk IS NULL AND cs_sold_date_sk=d_date_sk AND cs_item_sk=i_item_sk) foo GROUP BY channel, col_name, d_year, d_qoy, i_category ORDER BY channel, col_name, d_year, d_qoy, i_category LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q77.sql ================================================ -- CometBench-DS query 77 derived from TPC-DS query 77 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. with ss as (select s_store_sk, sum(ss_ext_sales_price) as sales, sum(ss_net_profit) as profit from store_sales, date_dim, store where ss_sold_date_sk = d_date_sk and d_date between cast('2001-08-11' as date) and (cast('2001-08-11' as date) + INTERVAL '30 DAYS') and ss_store_sk = s_store_sk group by s_store_sk) , sr as (select s_store_sk, sum(sr_return_amt) as returns, sum(sr_net_loss) as profit_loss from store_returns, date_dim, store where sr_returned_date_sk = d_date_sk and d_date between cast('2001-08-11' as date) and (cast('2001-08-11' as date) + INTERVAL '30 DAYS') and sr_store_sk = s_store_sk group by s_store_sk), cs as (select cs_call_center_sk, sum(cs_ext_sales_price) as sales, sum(cs_net_profit) as profit from catalog_sales, date_dim where cs_sold_date_sk = d_date_sk and d_date between cast('2001-08-11' as date) and (cast('2001-08-11' as date) + INTERVAL '30 DAYS') group by cs_call_center_sk ), cr as (select cr_call_center_sk, sum(cr_return_amount) as returns, sum(cr_net_loss) as profit_loss from catalog_returns, date_dim where cr_returned_date_sk = d_date_sk and d_date between cast('2001-08-11' as date) and (cast('2001-08-11' as date) + INTERVAL '30 DAYS') group by cr_call_center_sk ), ws as ( select wp_web_page_sk, sum(ws_ext_sales_price) as sales, sum(ws_net_profit) as profit from web_sales, date_dim, web_page where ws_sold_date_sk = d_date_sk and d_date between cast('2001-08-11' as date) and (cast('2001-08-11' as date) + INTERVAL '30 DAYS') and ws_web_page_sk = wp_web_page_sk group by wp_web_page_sk), wr as (select wp_web_page_sk, sum(wr_return_amt) as returns, sum(wr_net_loss) as profit_loss from web_returns, date_dim, web_page where wr_returned_date_sk = d_date_sk and d_date between cast('2001-08-11' as date) and (cast('2001-08-11' as date) + INTERVAL '30 DAYS') and wr_web_page_sk = wp_web_page_sk group by wp_web_page_sk) select channel , id , sum(sales) as sales , sum(returns) as returns , sum(profit) as profit from (select 'store channel' as channel , ss.s_store_sk as id , sales , coalesce(returns, 0) as returns , (profit - coalesce(profit_loss,0)) as profit from ss left join sr on ss.s_store_sk = sr.s_store_sk union all select 'catalog channel' as channel , cs_call_center_sk as id , sales , returns , (profit - profit_loss) as profit from cs , cr union all select 'web channel' as channel , ws.wp_web_page_sk as id , sales , coalesce(returns, 0) returns , (profit - coalesce(profit_loss,0)) as profit from ws left join wr on ws.wp_web_page_sk = wr.wp_web_page_sk ) x group by rollup (channel, id) order by channel ,id LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q78.sql ================================================ -- CometBench-DS query 78 derived from TPC-DS query 78 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. with ws as (select d_year AS ws_sold_year, ws_item_sk, ws_bill_customer_sk ws_customer_sk, sum(ws_quantity) ws_qty, sum(ws_wholesale_cost) ws_wc, sum(ws_sales_price) ws_sp from web_sales left join web_returns on wr_order_number=ws_order_number and ws_item_sk=wr_item_sk join date_dim on ws_sold_date_sk = d_date_sk where wr_order_number is null group by d_year, ws_item_sk, ws_bill_customer_sk ), cs as (select d_year AS cs_sold_year, cs_item_sk, cs_bill_customer_sk cs_customer_sk, sum(cs_quantity) cs_qty, sum(cs_wholesale_cost) cs_wc, sum(cs_sales_price) cs_sp from catalog_sales left join catalog_returns on cr_order_number=cs_order_number and cs_item_sk=cr_item_sk join date_dim on cs_sold_date_sk = d_date_sk where cr_order_number is null group by d_year, cs_item_sk, cs_bill_customer_sk ), ss as (select d_year AS ss_sold_year, ss_item_sk, ss_customer_sk, sum(ss_quantity) ss_qty, sum(ss_wholesale_cost) ss_wc, sum(ss_sales_price) ss_sp from store_sales left join store_returns on sr_ticket_number=ss_ticket_number and ss_item_sk=sr_item_sk join date_dim on ss_sold_date_sk = d_date_sk where sr_ticket_number is null group by d_year, ss_item_sk, ss_customer_sk ) select ss_customer_sk, round(ss_qty/(coalesce(ws_qty,0)+coalesce(cs_qty,0)),2) ratio, ss_qty store_qty, ss_wc store_wholesale_cost, ss_sp store_sales_price, coalesce(ws_qty,0)+coalesce(cs_qty,0) other_chan_qty, coalesce(ws_wc,0)+coalesce(cs_wc,0) other_chan_wholesale_cost, coalesce(ws_sp,0)+coalesce(cs_sp,0) other_chan_sales_price from ss left join ws on (ws_sold_year=ss_sold_year and ws_item_sk=ss_item_sk and ws_customer_sk=ss_customer_sk) left join cs on (cs_sold_year=ss_sold_year and cs_item_sk=ss_item_sk and cs_customer_sk=ss_customer_sk) where (coalesce(ws_qty,0)>0 or coalesce(cs_qty, 0)>0) and ss_sold_year=2001 order by ss_customer_sk, ss_qty desc, ss_wc desc, ss_sp desc, other_chan_qty, other_chan_wholesale_cost, other_chan_sales_price, ratio LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q79.sql ================================================ -- CometBench-DS query 79 derived from TPC-DS query 79 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select c_last_name,c_first_name,substr(s_city,1,30),ss_ticket_number,amt,profit from (select ss_ticket_number ,ss_customer_sk ,store.s_city ,sum(ss_coupon_amt) amt ,sum(ss_net_profit) profit from store_sales,date_dim,store,household_demographics where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_store_sk = store.s_store_sk and store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk and (household_demographics.hd_dep_count = 0 or household_demographics.hd_vehicle_count > 4) and date_dim.d_dow = 1 and date_dim.d_year in (1999,1999+1,1999+2) and store.s_number_employees between 200 and 295 group by ss_ticket_number,ss_customer_sk,ss_addr_sk,store.s_city) ms,customer where ss_customer_sk = c_customer_sk order by c_last_name,c_first_name,substr(s_city,1,30), profit LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q8.sql ================================================ -- CometBench-DS query 8 derived from TPC-DS query 8 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select s_store_name ,sum(ss_net_profit) from store_sales ,date_dim ,store, (select ca_zip from ( SELECT substr(ca_zip,1,5) ca_zip FROM customer_address WHERE substr(ca_zip,1,5) IN ( '19100','41548','51640','49699','88329','55986', '85119','19510','61020','95452','26235', '51102','16733','42819','27823','90192', '31905','28865','62197','23750','81398', '95288','45114','82060','12313','25218', '64386','46400','77230','69271','43672', '36521','34217','13017','27936','42766', '59233','26060','27477','39981','93402', '74270','13932','51731','71642','17710', '85156','21679','70840','67191','39214', '35273','27293','17128','15458','31615', '60706','67657','54092','32775','14683', '32206','62543','43053','11297','58216', '49410','14710','24501','79057','77038', '91286','32334','46298','18326','67213', '65382','40315','56115','80162','55956', '81583','73588','32513','62880','12201', '11592','17014','83832','61796','57872', '78829','69912','48524','22016','26905', '48511','92168','63051','25748','89786', '98827','86404','53029','37524','14039', '50078','34487','70142','18697','40129', '60642','42810','62667','57183','46414', '58463','71211','46364','34851','54884', '25382','25239','74126','21568','84204', '13607','82518','32982','36953','86001', '79278','21745','64444','35199','83181', '73255','86177','98043','90392','13882', '47084','17859','89526','42072','20233', '52745','75000','22044','77013','24182', '52554','56138','43440','86100','48791', '21883','17096','15965','31196','74903', '19810','35763','92020','55176','54433', '68063','71919','44384','16612','32109', '28207','14762','89933','10930','27616', '56809','14244','22733','33177','29784', '74968','37887','11299','34692','85843', '83663','95421','19323','17406','69264', '28341','50150','79121','73974','92917', '21229','32254','97408','46011','37169', '18146','27296','62927','68812','47734', '86572','12620','80252','50173','27261', '29534','23488','42184','23695','45868', '12910','23429','29052','63228','30731', '15747','25827','22332','62349','56661', '44652','51862','57007','22773','40361', '65238','19327','17282','44708','35484', '34064','11148','92729','22995','18833', '77528','48917','17256','93166','68576', '71096','56499','35096','80551','82424', '17700','32748','78969','46820','57725', '46179','54677','98097','62869','83959', '66728','19716','48326','27420','53458', '69056','84216','36688','63957','41469', '66843','18024','81950','21911','58387', '58103','19813','34581','55347','17171', '35914','75043','75088','80541','26802', '28849','22356','57721','77084','46385', '59255','29308','65885','70673','13306', '68788','87335','40987','31654','67560', '92309','78116','65961','45018','16548', '67092','21818','33716','49449','86150', '12156','27574','43201','50977','52839', '33234','86611','71494','17823','57172', '59869','34086','51052','11320','39717', '79604','24672','70555','38378','91135', '15567','21606','74994','77168','38607', '27384','68328','88944','40203','37893', '42726','83549','48739','55652','27543', '23109','98908','28831','45011','47525', '43870','79404','35780','42136','49317', '14574','99586','21107','14302','83882', '81272','92552','14916','87533','86518', '17862','30741','96288','57886','30304', '24201','79457','36728','49833','35182', '20108','39858','10804','47042','20439', '54708','59027','82499','75311','26548', '53406','92060','41152','60446','33129', '43979','16903','60319','35550','33887', '25463','40343','20726','44429') intersect select ca_zip from (SELECT substr(ca_zip,1,5) ca_zip,count(*) cnt FROM customer_address, customer WHERE ca_address_sk = c_current_addr_sk and c_preferred_cust_flag='Y' group by ca_zip having count(*) > 10)A1)A2) V1 where ss_store_sk = s_store_sk and ss_sold_date_sk = d_date_sk and d_qoy = 1 and d_year = 2000 and (substr(s_zip,1,2) = substr(V1.ca_zip,1,2)) group by s_store_name order by s_store_name LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q80.sql ================================================ -- CometBench-DS query 80 derived from TPC-DS query 80 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. with ssr as (select s_store_id as store_id, sum(ss_ext_sales_price) as sales, sum(coalesce(sr_return_amt, 0)) as returns, sum(ss_net_profit - coalesce(sr_net_loss, 0)) as profit from store_sales left outer join store_returns on (ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number), date_dim, store, item, promotion where ss_sold_date_sk = d_date_sk and d_date between cast('2002-08-04' as date) and (cast('2002-08-04' as date) + INTERVAL '30 DAYS') and ss_store_sk = s_store_sk and ss_item_sk = i_item_sk and i_current_price > 50 and ss_promo_sk = p_promo_sk and p_channel_tv = 'N' group by s_store_id) , csr as (select cp_catalog_page_id as catalog_page_id, sum(cs_ext_sales_price) as sales, sum(coalesce(cr_return_amount, 0)) as returns, sum(cs_net_profit - coalesce(cr_net_loss, 0)) as profit from catalog_sales left outer join catalog_returns on (cs_item_sk = cr_item_sk and cs_order_number = cr_order_number), date_dim, catalog_page, item, promotion where cs_sold_date_sk = d_date_sk and d_date between cast('2002-08-04' as date) and (cast('2002-08-04' as date) + INTERVAL '30 DAYS') and cs_catalog_page_sk = cp_catalog_page_sk and cs_item_sk = i_item_sk and i_current_price > 50 and cs_promo_sk = p_promo_sk and p_channel_tv = 'N' group by cp_catalog_page_id) , wsr as (select web_site_id, sum(ws_ext_sales_price) as sales, sum(coalesce(wr_return_amt, 0)) as returns, sum(ws_net_profit - coalesce(wr_net_loss, 0)) as profit from web_sales left outer join web_returns on (ws_item_sk = wr_item_sk and ws_order_number = wr_order_number), date_dim, web_site, item, promotion where ws_sold_date_sk = d_date_sk and d_date between cast('2002-08-04' as date) and (cast('2002-08-04' as date) + INTERVAL '30 DAYS') and ws_web_site_sk = web_site_sk and ws_item_sk = i_item_sk and i_current_price > 50 and ws_promo_sk = p_promo_sk and p_channel_tv = 'N' group by web_site_id) select channel , id , sum(sales) as sales , sum(returns) as returns , sum(profit) as profit from (select 'store channel' as channel , 'store' || store_id as id , sales , returns , profit from ssr union all select 'catalog channel' as channel , 'catalog_page' || catalog_page_id as id , sales , returns , profit from csr union all select 'web channel' as channel , 'web_site' || web_site_id as id , sales , returns , profit from wsr ) x group by rollup (channel, id) order by channel ,id LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q81.sql ================================================ -- CometBench-DS query 81 derived from TPC-DS query 81 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. with customer_total_return as (select cr_returning_customer_sk as ctr_customer_sk ,ca_state as ctr_state, sum(cr_return_amt_inc_tax) as ctr_total_return from catalog_returns ,date_dim ,customer_address where cr_returned_date_sk = d_date_sk and d_year =1998 and cr_returning_addr_sk = ca_address_sk group by cr_returning_customer_sk ,ca_state ) select c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name ,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset ,ca_location_type,ctr_total_return from customer_total_return ctr1 ,customer_address ,customer where ctr1.ctr_total_return > (select avg(ctr_total_return)*1.2 from customer_total_return ctr2 where ctr1.ctr_state = ctr2.ctr_state) and ca_address_sk = c_current_addr_sk and ca_state = 'TX' and ctr1.ctr_customer_sk = c_customer_sk order by c_customer_id,c_salutation,c_first_name,c_last_name,ca_street_number,ca_street_name ,ca_street_type,ca_suite_number,ca_city,ca_county,ca_state,ca_zip,ca_country,ca_gmt_offset ,ca_location_type,ctr_total_return LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q82.sql ================================================ -- CometBench-DS query 82 derived from TPC-DS query 82 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select i_item_id ,i_item_desc ,i_current_price from item, inventory, date_dim, store_sales where i_current_price between 69 and 69+30 and inv_item_sk = i_item_sk and d_date_sk=inv_date_sk and d_date between cast('1998-06-06' as date) and (cast('1998-06-06' as date) + INTERVAL '60 DAYS') and i_manufact_id in (105,513,180,137) and inv_quantity_on_hand between 100 and 500 and ss_item_sk = i_item_sk group by i_item_id,i_item_desc,i_current_price order by i_item_id LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q83.sql ================================================ -- CometBench-DS query 83 derived from TPC-DS query 83 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. with sr_items as (select i_item_id item_id, sum(sr_return_quantity) sr_item_qty from store_returns, item, date_dim where sr_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq in (select d_week_seq from date_dim where d_date in ('2000-04-29','2000-09-09','2000-11-02'))) and sr_returned_date_sk = d_date_sk group by i_item_id), cr_items as (select i_item_id item_id, sum(cr_return_quantity) cr_item_qty from catalog_returns, item, date_dim where cr_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq in (select d_week_seq from date_dim where d_date in ('2000-04-29','2000-09-09','2000-11-02'))) and cr_returned_date_sk = d_date_sk group by i_item_id), wr_items as (select i_item_id item_id, sum(wr_return_quantity) wr_item_qty from web_returns, item, date_dim where wr_item_sk = i_item_sk and d_date in (select d_date from date_dim where d_week_seq in (select d_week_seq from date_dim where d_date in ('2000-04-29','2000-09-09','2000-11-02'))) and wr_returned_date_sk = d_date_sk group by i_item_id) select sr_items.item_id ,sr_item_qty ,sr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 sr_dev ,cr_item_qty ,cr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 cr_dev ,wr_item_qty ,wr_item_qty/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 * 100 wr_dev ,(sr_item_qty+cr_item_qty+wr_item_qty)/3.0 average from sr_items ,cr_items ,wr_items where sr_items.item_id=cr_items.item_id and sr_items.item_id=wr_items.item_id order by sr_items.item_id ,sr_item_qty LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q84.sql ================================================ -- CometBench-DS query 84 derived from TPC-DS query 84 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select c_customer_id as customer_id , coalesce(c_last_name,'') || ', ' || coalesce(c_first_name,'') as customername from customer ,customer_address ,customer_demographics ,household_demographics ,income_band ,store_returns where ca_city = 'White Oak' and c_current_addr_sk = ca_address_sk and ib_lower_bound >= 45626 and ib_upper_bound <= 45626 + 50000 and ib_income_band_sk = hd_income_band_sk and cd_demo_sk = c_current_cdemo_sk and hd_demo_sk = c_current_hdemo_sk and sr_cdemo_sk = cd_demo_sk order by c_customer_id LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q85.sql ================================================ -- CometBench-DS query 85 derived from TPC-DS query 85 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select substr(r_reason_desc,1,20) ,avg(ws_quantity) ,avg(wr_refunded_cash) ,avg(wr_fee) from web_sales, web_returns, web_page, customer_demographics cd1, customer_demographics cd2, customer_address, date_dim, reason where ws_web_page_sk = wp_web_page_sk and ws_item_sk = wr_item_sk and ws_order_number = wr_order_number and ws_sold_date_sk = d_date_sk and d_year = 2001 and cd1.cd_demo_sk = wr_refunded_cdemo_sk and cd2.cd_demo_sk = wr_returning_cdemo_sk and ca_address_sk = wr_refunded_addr_sk and r_reason_sk = wr_reason_sk and ( ( cd1.cd_marital_status = 'D' and cd1.cd_marital_status = cd2.cd_marital_status and cd1.cd_education_status = 'Primary' and cd1.cd_education_status = cd2.cd_education_status and ws_sales_price between 100.00 and 150.00 ) or ( cd1.cd_marital_status = 'U' and cd1.cd_marital_status = cd2.cd_marital_status and cd1.cd_education_status = 'Unknown' and cd1.cd_education_status = cd2.cd_education_status and ws_sales_price between 50.00 and 100.00 ) or ( cd1.cd_marital_status = 'M' and cd1.cd_marital_status = cd2.cd_marital_status and cd1.cd_education_status = 'Advanced Degree' and cd1.cd_education_status = cd2.cd_education_status and ws_sales_price between 150.00 and 200.00 ) ) and ( ( ca_country = 'United States' and ca_state in ('SC', 'IN', 'VA') and ws_net_profit between 100 and 200 ) or ( ca_country = 'United States' and ca_state in ('WA', 'KS', 'KY') and ws_net_profit between 150 and 300 ) or ( ca_country = 'United States' and ca_state in ('SD', 'WI', 'NE') and ws_net_profit between 50 and 250 ) ) group by r_reason_desc order by substr(r_reason_desc,1,20) ,avg(ws_quantity) ,avg(wr_refunded_cash) ,avg(wr_fee) LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q86.sql ================================================ -- CometBench-DS query 86 derived from TPC-DS query 86 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select sum(ws_net_paid) as total_sum ,i_category ,i_class ,grouping(i_category)+grouping(i_class) as lochierarchy ,rank() over ( partition by grouping(i_category)+grouping(i_class), case when grouping(i_class) = 0 then i_category end order by sum(ws_net_paid) desc) as rank_within_parent from web_sales ,date_dim d1 ,item where d1.d_month_seq between 1205 and 1205+11 and d1.d_date_sk = ws_sold_date_sk and i_item_sk = ws_item_sk group by rollup(i_category,i_class) order by lochierarchy desc, case when lochierarchy = 0 then i_category end, rank_within_parent LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q87.sql ================================================ -- CometBench-DS query 87 derived from TPC-DS query 87 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select count(*) from ((select distinct c_last_name, c_first_name, d_date from store_sales, date_dim, customer where store_sales.ss_sold_date_sk = date_dim.d_date_sk and store_sales.ss_customer_sk = customer.c_customer_sk and d_month_seq between 1189 and 1189+11) except (select distinct c_last_name, c_first_name, d_date from catalog_sales, date_dim, customer where catalog_sales.cs_sold_date_sk = date_dim.d_date_sk and catalog_sales.cs_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1189 and 1189+11) except (select distinct c_last_name, c_first_name, d_date from web_sales, date_dim, customer where web_sales.ws_sold_date_sk = date_dim.d_date_sk and web_sales.ws_bill_customer_sk = customer.c_customer_sk and d_month_seq between 1189 and 1189+11) ) cool_cust ; ================================================ FILE: benchmarks/tpc/queries/tpcds/q88.sql ================================================ -- CometBench-DS query 88 derived from TPC-DS query 88 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select * from (select count(*) h8_30_to_9 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 8 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2)) and store.s_store_name = 'ese') s1, (select count(*) h9_to_9_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 9 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2)) and store.s_store_name = 'ese') s2, (select count(*) h9_30_to_10 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 9 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2)) and store.s_store_name = 'ese') s3, (select count(*) h10_to_10_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 10 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2)) and store.s_store_name = 'ese') s4, (select count(*) h10_30_to_11 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 10 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2)) and store.s_store_name = 'ese') s5, (select count(*) h11_to_11_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 11 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2)) and store.s_store_name = 'ese') s6, (select count(*) h11_30_to_12 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 11 and time_dim.t_minute >= 30 and ((household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2)) and store.s_store_name = 'ese') s7, (select count(*) h12_to_12_30 from store_sales, household_demographics , time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 12 and time_dim.t_minute < 30 and ((household_demographics.hd_dep_count = 2 and household_demographics.hd_vehicle_count<=2+2) or (household_demographics.hd_dep_count = 1 and household_demographics.hd_vehicle_count<=1+2) or (household_demographics.hd_dep_count = 4 and household_demographics.hd_vehicle_count<=4+2)) and store.s_store_name = 'ese') s8 ; ================================================ FILE: benchmarks/tpc/queries/tpcds/q89.sql ================================================ -- CometBench-DS query 89 derived from TPC-DS query 89 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select * from( select i_category, i_class, i_brand, s_store_name, s_company_name, d_moy, sum(ss_sales_price) sum_sales, avg(sum(ss_sales_price)) over (partition by i_category, i_brand, s_store_name, s_company_name) avg_monthly_sales from item, store_sales, date_dim, store where ss_item_sk = i_item_sk and ss_sold_date_sk = d_date_sk and ss_store_sk = s_store_sk and d_year in (2001) and ((i_category in ('Children','Jewelry','Home') and i_class in ('infants','birdal','flatware') ) or (i_category in ('Electronics','Music','Books') and i_class in ('audio','classical','science') )) group by i_category, i_class, i_brand, s_store_name, s_company_name, d_moy) tmp1 where case when (avg_monthly_sales <> 0) then (abs(sum_sales - avg_monthly_sales) / avg_monthly_sales) else null end > 0.1 order by sum_sales - avg_monthly_sales, s_store_name LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q9.sql ================================================ -- CometBench-DS query 9 derived from TPC-DS query 9 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select case when (select count(*) from store_sales where ss_quantity between 1 and 20) > 31002 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 1 and 20) else (select avg(ss_net_profit) from store_sales where ss_quantity between 1 and 20) end bucket1 , case when (select count(*) from store_sales where ss_quantity between 21 and 40) > 588 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 21 and 40) else (select avg(ss_net_profit) from store_sales where ss_quantity between 21 and 40) end bucket2, case when (select count(*) from store_sales where ss_quantity between 41 and 60) > 2456 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 41 and 60) else (select avg(ss_net_profit) from store_sales where ss_quantity between 41 and 60) end bucket3, case when (select count(*) from store_sales where ss_quantity between 61 and 80) > 21645 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 61 and 80) else (select avg(ss_net_profit) from store_sales where ss_quantity between 61 and 80) end bucket4, case when (select count(*) from store_sales where ss_quantity between 81 and 100) > 20553 then (select avg(ss_ext_discount_amt) from store_sales where ss_quantity between 81 and 100) else (select avg(ss_net_profit) from store_sales where ss_quantity between 81 and 100) end bucket5 from reason where r_reason_sk = 1 ; ================================================ FILE: benchmarks/tpc/queries/tpcds/q90.sql ================================================ -- CometBench-DS query 90 derived from TPC-DS query 90 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select cast(amc as decimal(15,4))/cast(pmc as decimal(15,4)) am_pm_ratio from ( select count(*) amc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 9 and 9+1 and household_demographics.hd_dep_count = 2 and web_page.wp_char_count between 5000 and 5200) at, ( select count(*) pmc from web_sales, household_demographics , time_dim, web_page where ws_sold_time_sk = time_dim.t_time_sk and ws_ship_hdemo_sk = household_demographics.hd_demo_sk and ws_web_page_sk = web_page.wp_web_page_sk and time_dim.t_hour between 15 and 15+1 and household_demographics.hd_dep_count = 2 and web_page.wp_char_count between 5000 and 5200) pt order by am_pm_ratio LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q91.sql ================================================ -- CometBench-DS query 91 derived from TPC-DS query 91 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select cc_call_center_id Call_Center, cc_name Call_Center_Name, cc_manager Manager, sum(cr_net_loss) Returns_Loss from call_center, catalog_returns, date_dim, customer, customer_address, customer_demographics, household_demographics where cr_call_center_sk = cc_call_center_sk and cr_returned_date_sk = d_date_sk and cr_returning_customer_sk= c_customer_sk and cd_demo_sk = c_current_cdemo_sk and hd_demo_sk = c_current_hdemo_sk and ca_address_sk = c_current_addr_sk and d_year = 2002 and d_moy = 11 and ( (cd_marital_status = 'M' and cd_education_status = 'Unknown') or(cd_marital_status = 'W' and cd_education_status = 'Advanced Degree')) and hd_buy_potential like 'Unknown%' and ca_gmt_offset = -6 group by cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status order by sum(cr_net_loss) desc; ================================================ FILE: benchmarks/tpc/queries/tpcds/q92.sql ================================================ -- CometBench-DS query 92 derived from TPC-DS query 92 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select sum(ws_ext_discount_amt) as `Excess Discount Amount` from web_sales ,item ,date_dim where i_manufact_id = 914 and i_item_sk = ws_item_sk and d_date between '2001-01-25' and (cast('2001-01-25' as date) + INTERVAL '90 DAYS') and d_date_sk = ws_sold_date_sk and ws_ext_discount_amt > ( SELECT 1.3 * avg(ws_ext_discount_amt) FROM web_sales ,date_dim WHERE ws_item_sk = i_item_sk and d_date between '2001-01-25' and (cast('2001-01-25' as date) + INTERVAL '90 DAYS') and d_date_sk = ws_sold_date_sk ) order by sum(ws_ext_discount_amt) LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q93.sql ================================================ -- CometBench-DS query 93 derived from TPC-DS query 93 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select ss_customer_sk ,sum(act_sales) sumsales from (select ss_item_sk ,ss_ticket_number ,ss_customer_sk ,case when sr_return_quantity is not null then (ss_quantity-sr_return_quantity)*ss_sales_price else (ss_quantity*ss_sales_price) end act_sales from store_sales left outer join store_returns on (sr_item_sk = ss_item_sk and sr_ticket_number = ss_ticket_number) ,reason where sr_reason_sk = r_reason_sk and r_reason_desc = 'Did not get it on time') t group by ss_customer_sk order by sumsales, ss_customer_sk LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q94.sql ================================================ -- CometBench-DS query 94 derived from TPC-DS query 94 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select count(distinct ws_order_number) as `order count` ,sum(ws_ext_ship_cost) as `total shipping cost` ,sum(ws_net_profit) as `total net profit` from web_sales ws1 ,date_dim ,customer_address ,web_site where d_date between '1999-4-01' and (cast('1999-4-01' as date) + INTERVAL '60 DAYS') and ws1.ws_ship_date_sk = d_date_sk and ws1.ws_ship_addr_sk = ca_address_sk and ca_state = 'WI' and ws1.ws_web_site_sk = web_site_sk and web_company_name = 'pri' and exists (select * from web_sales ws2 where ws1.ws_order_number = ws2.ws_order_number and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) and not exists(select * from web_returns wr1 where ws1.ws_order_number = wr1.wr_order_number) order by count(distinct ws_order_number) LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q95.sql ================================================ -- CometBench-DS query 95 derived from TPC-DS query 95 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. with ws_wh as (select ws1.ws_order_number,ws1.ws_warehouse_sk wh1,ws2.ws_warehouse_sk wh2 from web_sales ws1,web_sales ws2 where ws1.ws_order_number = ws2.ws_order_number and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) select count(distinct ws_order_number) as `order count` ,sum(ws_ext_ship_cost) as `total shipping cost` ,sum(ws_net_profit) as `total net profit` from web_sales ws1 ,date_dim ,customer_address ,web_site where d_date between '2002-5-01' and (cast('2002-5-01' as date) + INTERVAL '60 DAYS') and ws1.ws_ship_date_sk = d_date_sk and ws1.ws_ship_addr_sk = ca_address_sk and ca_state = 'MA' and ws1.ws_web_site_sk = web_site_sk and web_company_name = 'pri' and ws1.ws_order_number in (select ws_order_number from ws_wh) and ws1.ws_order_number in (select wr_order_number from web_returns,ws_wh where wr_order_number = ws_wh.ws_order_number) order by count(distinct ws_order_number) LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q96.sql ================================================ -- CometBench-DS query 96 derived from TPC-DS query 96 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select count(*) from store_sales ,household_demographics ,time_dim, store where ss_sold_time_sk = time_dim.t_time_sk and ss_hdemo_sk = household_demographics.hd_demo_sk and ss_store_sk = s_store_sk and time_dim.t_hour = 8 and time_dim.t_minute >= 30 and household_demographics.hd_dep_count = 5 and store.s_store_name = 'ese' order by count(*) LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q97.sql ================================================ -- CometBench-DS query 97 derived from TPC-DS query 97 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. with ssci as ( select ss_customer_sk customer_sk ,ss_item_sk item_sk from store_sales,date_dim where ss_sold_date_sk = d_date_sk and d_month_seq between 1211 and 1211 + 11 group by ss_customer_sk ,ss_item_sk), csci as( select cs_bill_customer_sk customer_sk ,cs_item_sk item_sk from catalog_sales,date_dim where cs_sold_date_sk = d_date_sk and d_month_seq between 1211 and 1211 + 11 group by cs_bill_customer_sk ,cs_item_sk) select sum(case when ssci.customer_sk is not null and csci.customer_sk is null then 1 else 0 end) store_only ,sum(case when ssci.customer_sk is null and csci.customer_sk is not null then 1 else 0 end) catalog_only ,sum(case when ssci.customer_sk is not null and csci.customer_sk is not null then 1 else 0 end) store_and_catalog from ssci full outer join csci on (ssci.customer_sk=csci.customer_sk and ssci.item_sk = csci.item_sk) LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpcds/q98.sql ================================================ -- CometBench-DS query 98 derived from TPC-DS query 98 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price ,sum(ss_ext_sales_price) as itemrevenue ,sum(ss_ext_sales_price)*100/sum(sum(ss_ext_sales_price)) over (partition by i_class) as revenueratio from store_sales ,item ,date_dim where ss_item_sk = i_item_sk and i_category in ('Shoes', 'Music', 'Men') and ss_sold_date_sk = d_date_sk and d_date between cast('2000-01-05' as date) and (cast('2000-01-05' as date) + INTERVAL '30 DAYS') group by i_item_id ,i_item_desc ,i_category ,i_class ,i_current_price order by i_category ,i_class ,i_item_id ,i_item_desc ,revenueratio; ================================================ FILE: benchmarks/tpc/queries/tpcds/q99.sql ================================================ -- CometBench-DS query 99 derived from TPC-DS query 99 under the terms of the TPC Fair Use Policy. -- TPC-DS queries are Copyright 2021 Transaction Processing Performance Council. -- This query was generated at scale factor 1. select substr(w_warehouse_name,1,20) ,sm_type ,cc_name ,sum(case when (cs_ship_date_sk - cs_sold_date_sk <= 30 ) then 1 else 0 end) as `30 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 30) and (cs_ship_date_sk - cs_sold_date_sk <= 60) then 1 else 0 end ) as `31-60 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 60) and (cs_ship_date_sk - cs_sold_date_sk <= 90) then 1 else 0 end) as `61-90 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 90) and (cs_ship_date_sk - cs_sold_date_sk <= 120) then 1 else 0 end) as `91-120 days` ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 120) then 1 else 0 end) as `>120 days` from catalog_sales ,warehouse ,ship_mode ,call_center ,date_dim where d_month_seq between 1188 and 1188 + 11 and cs_ship_date_sk = d_date_sk and cs_warehouse_sk = w_warehouse_sk and cs_ship_mode_sk = sm_ship_mode_sk and cs_call_center_sk = cc_call_center_sk group by substr(w_warehouse_name,1,20) ,sm_type ,cc_name order by substr(w_warehouse_name,1,20) ,sm_type ,cc_name LIMIT 100; ================================================ FILE: benchmarks/tpc/queries/tpch/q1.sql ================================================ -- CometBench-H query 1 derived from TPC-H query 1 under the terms of the TPC Fair Use Policy. -- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. select l_returnflag, l_linestatus, sum(l_quantity) as sum_qty, sum(l_extendedprice) as sum_base_price, sum(l_extendedprice * (1 - l_discount)) as sum_disc_price, sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) as sum_charge, avg(l_quantity) as avg_qty, avg(l_extendedprice) as avg_price, avg(l_discount) as avg_disc, count(*) as count_order from lineitem where l_shipdate <= date '1998-12-01' - interval '68 days' group by l_returnflag, l_linestatus order by l_returnflag, l_linestatus; ================================================ FILE: benchmarks/tpc/queries/tpch/q10.sql ================================================ -- CometBench-H query 10 derived from TPC-H query 10 under the terms of the TPC Fair Use Policy. -- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. select c_custkey, c_name, sum(l_extendedprice * (1 - l_discount)) as revenue, c_acctbal, n_name, c_address, c_phone, c_comment from customer, orders, lineitem, nation where c_custkey = o_custkey and l_orderkey = o_orderkey and o_orderdate >= date '1993-07-01' and o_orderdate < date '1993-07-01' + interval '3' month and l_returnflag = 'R' and c_nationkey = n_nationkey group by c_custkey, c_name, c_acctbal, c_phone, n_name, c_address, c_comment order by revenue desc limit 20; ================================================ FILE: benchmarks/tpc/queries/tpch/q11.sql ================================================ -- CometBench-H query 11 derived from TPC-H query 11 under the terms of the TPC Fair Use Policy. -- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. select ps_partkey, sum(ps_supplycost * ps_availqty) as value from partsupp, supplier, nation where ps_suppkey = s_suppkey and s_nationkey = n_nationkey and n_name = 'ALGERIA' group by ps_partkey having sum(ps_supplycost * ps_availqty) > ( select sum(ps_supplycost * ps_availqty) * 0.0001000000 from partsupp, supplier, nation where ps_suppkey = s_suppkey and s_nationkey = n_nationkey and n_name = 'ALGERIA' ) order by value desc; ================================================ FILE: benchmarks/tpc/queries/tpch/q12.sql ================================================ -- CometBench-H query 12 derived from TPC-H query 12 under the terms of the TPC Fair Use Policy. -- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. select l_shipmode, sum(case when o_orderpriority = '1-URGENT' or o_orderpriority = '2-HIGH' then 1 else 0 end) as high_line_count, sum(case when o_orderpriority <> '1-URGENT' and o_orderpriority <> '2-HIGH' then 1 else 0 end) as low_line_count from orders, lineitem where o_orderkey = l_orderkey and l_shipmode in ('FOB', 'SHIP') and l_commitdate < l_receiptdate and l_shipdate < l_commitdate and l_receiptdate >= date '1995-01-01' and l_receiptdate < date '1995-01-01' + interval '1' year group by l_shipmode order by l_shipmode; ================================================ FILE: benchmarks/tpc/queries/tpch/q13.sql ================================================ -- CometBench-H query 13 derived from TPC-H query 13 under the terms of the TPC Fair Use Policy. -- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. select c_count, count(*) as custdist from ( select c_custkey, count(o_orderkey) from customer left outer join orders on c_custkey = o_custkey and o_comment not like '%express%requests%' group by c_custkey ) as c_orders (c_custkey, c_count) group by c_count order by custdist desc, c_count desc; ================================================ FILE: benchmarks/tpc/queries/tpch/q14.sql ================================================ -- CometBench-H query 14 derived from TPC-H query 14 under the terms of the TPC Fair Use Policy. -- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. select 100.00 * sum(case when p_type like 'PROMO%' then l_extendedprice * (1 - l_discount) else 0 end) / sum(l_extendedprice * (1 - l_discount)) as promo_revenue from lineitem, part where l_partkey = p_partkey and l_shipdate >= date '1995-02-01' and l_shipdate < date '1995-02-01' + interval '1' month; ================================================ FILE: benchmarks/tpc/queries/tpch/q15.sql ================================================ -- CometBench-H query 15 derived from TPC-H query 15 under the terms of the TPC Fair Use Policy. -- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. create view revenue0 (supplier_no, total_revenue) as select l_suppkey, sum(l_extendedprice * (1 - l_discount)) from lineitem where l_shipdate >= date '1996-08-01' and l_shipdate < date '1996-08-01' + interval '3' month group by l_suppkey; select s_suppkey, s_name, s_address, s_phone, total_revenue from supplier, revenue0 where s_suppkey = supplier_no and total_revenue = ( select max(total_revenue) from revenue0 ) order by s_suppkey; drop view revenue0; ================================================ FILE: benchmarks/tpc/queries/tpch/q16.sql ================================================ -- CometBench-H query 16 derived from TPC-H query 16 under the terms of the TPC Fair Use Policy. -- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. select p_brand, p_type, p_size, count(distinct ps_suppkey) as supplier_cnt from partsupp, part where p_partkey = ps_partkey and p_brand <> 'Brand#14' and p_type not like 'SMALL PLATED%' and p_size in (14, 6, 5, 31, 49, 15, 41, 47) and ps_suppkey not in ( select s_suppkey from supplier where s_comment like '%Customer%Complaints%' ) group by p_brand, p_type, p_size order by supplier_cnt desc, p_brand, p_type, p_size; ================================================ FILE: benchmarks/tpc/queries/tpch/q17.sql ================================================ -- CometBench-H query 17 derived from TPC-H query 17 under the terms of the TPC Fair Use Policy. -- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. select sum(l_extendedprice) / 7.0 as avg_yearly from lineitem, part where p_partkey = l_partkey and p_brand = 'Brand#42' and p_container = 'LG BAG' and l_quantity < ( select 0.2 * avg(l_quantity) from lineitem where l_partkey = p_partkey ); ================================================ FILE: benchmarks/tpc/queries/tpch/q18.sql ================================================ -- CometBench-H query 18 derived from TPC-H query 18 under the terms of the TPC Fair Use Policy. -- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. select c_name, c_custkey, o_orderkey, o_orderdate, o_totalprice, sum(l_quantity) from customer, orders, lineitem where o_orderkey in ( select l_orderkey from lineitem group by l_orderkey having sum(l_quantity) > 313 ) and c_custkey = o_custkey and o_orderkey = l_orderkey group by c_name, c_custkey, o_orderkey, o_orderdate, o_totalprice order by o_totalprice desc, o_orderdate limit 100; ================================================ FILE: benchmarks/tpc/queries/tpch/q19.sql ================================================ -- CometBench-H query 19 derived from TPC-H query 19 under the terms of the TPC Fair Use Policy. -- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. select sum(l_extendedprice* (1 - l_discount)) as revenue from lineitem, part where ( p_partkey = l_partkey and p_brand = 'Brand#21' and p_container in ('SM CASE', 'SM BOX', 'SM PACK', 'SM PKG') and l_quantity >= 8 and l_quantity <= 8 + 10 and p_size between 1 and 5 and l_shipmode in ('AIR', 'AIR REG') and l_shipinstruct = 'DELIVER IN PERSON' ) or ( p_partkey = l_partkey and p_brand = 'Brand#13' and p_container in ('MED BAG', 'MED BOX', 'MED PKG', 'MED PACK') and l_quantity >= 20 and l_quantity <= 20 + 10 and p_size between 1 and 10 and l_shipmode in ('AIR', 'AIR REG') and l_shipinstruct = 'DELIVER IN PERSON' ) or ( p_partkey = l_partkey and p_brand = 'Brand#52' and p_container in ('LG CASE', 'LG BOX', 'LG PACK', 'LG PKG') and l_quantity >= 30 and l_quantity <= 30 + 10 and p_size between 1 and 15 and l_shipmode in ('AIR', 'AIR REG') and l_shipinstruct = 'DELIVER IN PERSON' ); ================================================ FILE: benchmarks/tpc/queries/tpch/q2.sql ================================================ -- CometBench-H query 2 derived from TPC-H query 2 under the terms of the TPC Fair Use Policy. -- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. select s_acctbal, s_name, n_name, p_partkey, p_mfgr, s_address, s_phone, s_comment from part, supplier, partsupp, nation, region where p_partkey = ps_partkey and s_suppkey = ps_suppkey and p_size = 48 and p_type like '%TIN' and s_nationkey = n_nationkey and n_regionkey = r_regionkey and r_name = 'ASIA' and ps_supplycost = ( select min(ps_supplycost) from partsupp, supplier, nation, region where p_partkey = ps_partkey and s_suppkey = ps_suppkey and s_nationkey = n_nationkey and n_regionkey = r_regionkey and r_name = 'ASIA' ) order by s_acctbal desc, n_name, s_name, p_partkey limit 100; ================================================ FILE: benchmarks/tpc/queries/tpch/q20.sql ================================================ -- CometBench-H query 20 derived from TPC-H query 20 under the terms of the TPC Fair Use Policy. -- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. select s_name, s_address from supplier, nation where s_suppkey in ( select ps_suppkey from partsupp where ps_partkey in ( select p_partkey from part where p_name like 'blanched%' ) and ps_availqty > ( select 0.5 * sum(l_quantity) from lineitem where l_partkey = ps_partkey and l_suppkey = ps_suppkey and l_shipdate >= date '1993-01-01' and l_shipdate < date '1993-01-01' + interval '1' year ) ) and s_nationkey = n_nationkey and n_name = 'KENYA' order by s_name; ================================================ FILE: benchmarks/tpc/queries/tpch/q21.sql ================================================ -- CometBench-H query 21 derived from TPC-H query 21 under the terms of the TPC Fair Use Policy. -- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. select s_name, count(*) as numwait from supplier, lineitem l1, orders, nation where s_suppkey = l1.l_suppkey and o_orderkey = l1.l_orderkey and o_orderstatus = 'F' and l1.l_receiptdate > l1.l_commitdate and exists ( select * from lineitem l2 where l2.l_orderkey = l1.l_orderkey and l2.l_suppkey <> l1.l_suppkey ) and not exists ( select * from lineitem l3 where l3.l_orderkey = l1.l_orderkey and l3.l_suppkey <> l1.l_suppkey and l3.l_receiptdate > l3.l_commitdate ) and s_nationkey = n_nationkey and n_name = 'ARGENTINA' group by s_name order by numwait desc, s_name limit 100; ================================================ FILE: benchmarks/tpc/queries/tpch/q22.sql ================================================ -- CometBench-H query 22 derived from TPC-H query 22 under the terms of the TPC Fair Use Policy. -- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. select cntrycode, count(*) as numcust, sum(c_acctbal) as totacctbal from ( select substring(c_phone from 1 for 2) as cntrycode, c_acctbal from customer where substring(c_phone from 1 for 2) in ('24', '34', '16', '30', '33', '14', '13') and c_acctbal > ( select avg(c_acctbal) from customer where c_acctbal > 0.00 and substring(c_phone from 1 for 2) in ('24', '34', '16', '30', '33', '14', '13') ) and not exists ( select * from orders where o_custkey = c_custkey ) ) as custsale group by cntrycode order by cntrycode; ================================================ FILE: benchmarks/tpc/queries/tpch/q3.sql ================================================ -- CometBench-H query 3 derived from TPC-H query 3 under the terms of the TPC Fair Use Policy. -- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. select l_orderkey, sum(l_extendedprice * (1 - l_discount)) as revenue, o_orderdate, o_shippriority from customer, orders, lineitem where c_mktsegment = 'BUILDING' and c_custkey = o_custkey and l_orderkey = o_orderkey and o_orderdate < date '1995-03-15' and l_shipdate > date '1995-03-15' group by l_orderkey, o_orderdate, o_shippriority order by revenue desc, o_orderdate limit 10; ================================================ FILE: benchmarks/tpc/queries/tpch/q4.sql ================================================ -- CometBench-H query 4 derived from TPC-H query 4 under the terms of the TPC Fair Use Policy. -- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. select o_orderpriority, count(*) as order_count from orders where o_orderdate >= date '1995-04-01' and o_orderdate < date '1995-04-01' + interval '3' month and exists ( select * from lineitem where l_orderkey = o_orderkey and l_commitdate < l_receiptdate ) group by o_orderpriority order by o_orderpriority; ================================================ FILE: benchmarks/tpc/queries/tpch/q5.sql ================================================ -- CometBench-H query 5 derived from TPC-H query 5 under the terms of the TPC Fair Use Policy. -- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. select n_name, sum(l_extendedprice * (1 - l_discount)) as revenue from customer, orders, lineitem, supplier, nation, region where c_custkey = o_custkey and l_orderkey = o_orderkey and l_suppkey = s_suppkey and c_nationkey = s_nationkey and s_nationkey = n_nationkey and n_regionkey = r_regionkey and r_name = 'AFRICA' and o_orderdate >= date '1994-01-01' and o_orderdate < date '1994-01-01' + interval '1' year group by n_name order by revenue desc; ================================================ FILE: benchmarks/tpc/queries/tpch/q6.sql ================================================ -- CometBench-H query 6 derived from TPC-H query 6 under the terms of the TPC Fair Use Policy. -- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. select sum(l_extendedprice * l_discount) as revenue from lineitem where l_shipdate >= date '1994-01-01' and l_shipdate < date '1994-01-01' + interval '1' year and l_discount between 0.04 - 0.01 and 0.04 + 0.01 and l_quantity < 24; ================================================ FILE: benchmarks/tpc/queries/tpch/q7.sql ================================================ -- CometBench-H query 7 derived from TPC-H query 7 under the terms of the TPC Fair Use Policy. -- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. select supp_nation, cust_nation, l_year, sum(volume) as revenue from ( select n1.n_name as supp_nation, n2.n_name as cust_nation, extract(year from l_shipdate) as l_year, l_extendedprice * (1 - l_discount) as volume from supplier, lineitem, orders, customer, nation n1, nation n2 where s_suppkey = l_suppkey and o_orderkey = l_orderkey and c_custkey = o_custkey and s_nationkey = n1.n_nationkey and c_nationkey = n2.n_nationkey and ( (n1.n_name = 'GERMANY' and n2.n_name = 'IRAQ') or (n1.n_name = 'IRAQ' and n2.n_name = 'GERMANY') ) and l_shipdate between date '1995-01-01' and date '1996-12-31' ) as shipping group by supp_nation, cust_nation, l_year order by supp_nation, cust_nation, l_year; ================================================ FILE: benchmarks/tpc/queries/tpch/q8.sql ================================================ -- CometBench-H query 8 derived from TPC-H query 8 under the terms of the TPC Fair Use Policy. -- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. select o_year, sum(case when nation = 'IRAQ' then volume else 0 end) / sum(volume) as mkt_share from ( select extract(year from o_orderdate) as o_year, l_extendedprice * (1 - l_discount) as volume, n2.n_name as nation from part, supplier, lineitem, orders, customer, nation n1, nation n2, region where p_partkey = l_partkey and s_suppkey = l_suppkey and l_orderkey = o_orderkey and o_custkey = c_custkey and c_nationkey = n1.n_nationkey and n1.n_regionkey = r_regionkey and r_name = 'MIDDLE EAST' and s_nationkey = n2.n_nationkey and o_orderdate between date '1995-01-01' and date '1996-12-31' and p_type = 'LARGE PLATED STEEL' ) as all_nations group by o_year order by o_year; ================================================ FILE: benchmarks/tpc/queries/tpch/q9.sql ================================================ -- CometBench-H query 9 derived from TPC-H query 9 under the terms of the TPC Fair Use Policy. -- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. select nation, o_year, sum(amount) as sum_profit from ( select n_name as nation, extract(year from o_orderdate) as o_year, l_extendedprice * (1 - l_discount) - ps_supplycost * l_quantity as amount from part, supplier, lineitem, partsupp, orders, nation where s_suppkey = l_suppkey and ps_suppkey = l_suppkey and ps_partkey = l_partkey and p_partkey = l_partkey and o_orderkey = l_orderkey and s_nationkey = n_nationkey and p_name like '%moccasin%' ) as profit group by nation, o_year order by nation, o_year desc; ================================================ FILE: benchmarks/tpc/run.py ================================================ #!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. """Consolidated TPC benchmark runner for Spark-based engines. Usage: python3 run.py --engine comet --benchmark tpch python3 run.py --engine comet --benchmark tpcds --iterations 3 python3 run.py --engine comet-iceberg --benchmark tpch python3 run.py --engine comet --benchmark tpch --dry-run python3 run.py --engine spark --benchmark tpch --no-restart """ import argparse import os import re import subprocess import sys # --------------------------------------------------------------------------- # TOML loading – prefer stdlib tomllib (3.11+), else minimal fallback # --------------------------------------------------------------------------- try: import tomllib # Python 3.11+ def load_toml(path): with open(path, "rb") as f: return tomllib.load(f) except ModuleNotFoundError: def _parse_toml(text): """Minimal TOML parser supporting tables, quoted-key strings, plain strings, arrays of strings, booleans, and comments. Sufficient for the engine config files used by this runner.""" root = {} current = root for line in text.splitlines(): line = line.strip() if not line or line.startswith("#"): continue # Table header: [env.defaults] or [spark_conf] m = re.match(r"^\[([^\]]+)\]$", line) if m: keys = m.group(1).split(".") current = root for k in keys: current = current.setdefault(k, {}) continue # Key = value m = re.match(r'^("(?:[^"\\]|\\.)*"|[A-Za-z0-9_.]+)\s*=\s*(.+)$', line) if not m: continue raw_key, raw_val = m.group(1), m.group(2).strip() key = raw_key.strip('"') val = _parse_value(raw_val) current[key] = val return root def _parse_value(raw): if raw == "true": return True if raw == "false": return False if raw.startswith('"') and raw.endswith('"'): return raw[1:-1] if raw.startswith("["): # Simple array of strings items = [] for m in re.finditer(r'"((?:[^"\\]|\\.)*)"', raw): items.append(m.group(1)) return items if raw.startswith("{"): # Inline table: { KEY = "VAL", ... } tbl = {} for m in re.finditer(r'([A-Za-z0-9_]+)\s*=\s*"((?:[^"\\]|\\.)*)"', raw): tbl[m.group(1)] = m.group(2) return tbl return raw def load_toml(path): with open(path, "r") as f: return _parse_toml(f.read()) # --------------------------------------------------------------------------- # Common Spark configuration (shared across all engines) # --------------------------------------------------------------------------- COMMON_SPARK_CONF = { "spark.driver.memory": "8G", "spark.executor.memory": "16g", "spark.memory.offHeap.enabled": "true", "spark.memory.offHeap.size": "16g", "spark.eventLog.enabled": "true", "spark.eventLog.dir": os.environ.get("SPARK_EVENT_LOG_DIR", "/tmp/spark-events"), "spark.hadoop.fs.s3a.impl": "org.apache.hadoop.fs.s3a.S3AFileSystem", "spark.hadoop.fs.s3a.aws.credentials.provider": "com.amazonaws.auth.DefaultAWSCredentialsProviderChain", } # --------------------------------------------------------------------------- # Benchmark profiles # --------------------------------------------------------------------------- BENCHMARK_PROFILES = { "tpch": { "executor_instances": "2", "executor_cores": "8", "max_cores": "16", "data_env": "TPCH_DATA", "format": "parquet", }, "tpcds": { "executor_instances": "2", "executor_cores": "8", "max_cores": "16", "data_env": "TPCDS_DATA", "format": None, # omit --format for TPC-DS }, } # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- def resolve_env(value): """Expand $VAR and ${VAR} references using os.environ.""" if not isinstance(value, str): return value return re.sub( r"\$\{([^}]+)\}|\$([A-Za-z_][A-Za-z0-9_]*)", lambda m: os.environ.get(m.group(1) or m.group(2), ""), value, ) def resolve_env_in_list(lst): return [resolve_env(v) for v in lst] def load_engine_config(engine_name): """Load and return the TOML config for the given engine.""" script_dir = os.path.dirname(os.path.abspath(__file__)) config_path = os.path.join(script_dir, "engines", f"{engine_name}.toml") if not os.path.exists(config_path): available = sorted( f.removesuffix(".toml") for f in os.listdir(os.path.join(script_dir, "engines")) if f.endswith(".toml") ) print(f"Error: Unknown engine '{engine_name}'", file=sys.stderr) print(f"Available engines: {', '.join(available)}", file=sys.stderr) sys.exit(1) return load_toml(config_path) def apply_env_defaults(config): """Set environment variable defaults from [env.defaults].""" defaults = config.get("env", {}).get("defaults", {}) for key, val in defaults.items(): if key not in os.environ: os.environ[key] = val def apply_env_exports(config): """Export environment variables from [env.exports].""" exports = config.get("env", {}).get("exports", {}) for key, val in exports.items(): os.environ[key] = val def check_required_env(config): """Validate that required environment variables are set.""" required = config.get("env", {}).get("required", []) missing = [v for v in required if not os.environ.get(v)] if missing: print( f"Error: Required environment variable(s) not set: {', '.join(missing)}", file=sys.stderr, ) sys.exit(1) def check_common_env(): """Validate SPARK_HOME and SPARK_MASTER are set.""" for var in ("SPARK_HOME", "SPARK_MASTER"): if not os.environ.get(var): print(f"Error: {var} is not set", file=sys.stderr) sys.exit(1) def check_benchmark_env(config, benchmark): """Validate benchmark-specific environment variables.""" profile = BENCHMARK_PROFILES[benchmark] use_iceberg = config.get("tpcbench_args", {}).get("use_iceberg", False) required = [] if not use_iceberg: required.append(profile["data_env"]) missing = [v for v in required if not os.environ.get(v)] if missing: print( f"Error: Required environment variable(s) not set for " f"{benchmark}: {', '.join(missing)}", file=sys.stderr, ) sys.exit(1) # Default ICEBERG_DATABASE to the benchmark name if not already set if use_iceberg and not os.environ.get("ICEBERG_DATABASE"): os.environ["ICEBERG_DATABASE"] = benchmark def build_spark_submit_cmd(config, benchmark, args): """Build the spark-submit command list.""" spark_home = os.environ["SPARK_HOME"] spark_master = os.environ["SPARK_MASTER"] profile = BENCHMARK_PROFILES[benchmark] cmd = [os.path.join(spark_home, "bin", "spark-submit")] cmd += ["--master", spark_master] # --jars jars = config.get("spark_submit", {}).get("jars", []) if jars: cmd += ["--jars", ",".join(resolve_env_in_list(jars))] # --driver-class-path driver_cp = config.get("spark_submit", {}).get("driver_class_path", []) if driver_cp: cmd += ["--driver-class-path", ":".join(resolve_env_in_list(driver_cp))] # Merge spark confs: common + benchmark profile + engine overrides conf = dict(COMMON_SPARK_CONF) conf["spark.executor.instances"] = profile["executor_instances"] conf["spark.executor.cores"] = profile["executor_cores"] conf["spark.cores.max"] = profile["max_cores"] engine_conf = config.get("spark_conf", {}) for key, val in engine_conf.items(): if isinstance(val, bool): val = "true" if val else "false" conf[resolve_env(key)] = resolve_env(str(val)) # JFR profiling: append to extraJavaOptions (preserving any existing values) if args.jfr: jfr_dir = args.jfr_dir driver_jfr = ( f"-XX:StartFlightRecording=disk=true,dumponexit=true," f"filename={jfr_dir}/driver.jfr,settings=profile" ) executor_jfr = ( f"-XX:StartFlightRecording=disk=true,dumponexit=true," f"filename={jfr_dir}/executor.jfr,settings=profile" ) for spark_key, jfr_opts in [ ("spark.driver.extraJavaOptions", driver_jfr), ("spark.executor.extraJavaOptions", executor_jfr), ]: existing = conf.get(spark_key, "") conf[spark_key] = f"{existing} {jfr_opts}".strip() # async-profiler: attach as a Java agent via -agentpath if args.async_profiler: ap_home = os.environ.get("ASYNC_PROFILER_HOME", "") if not ap_home: print( "Error: ASYNC_PROFILER_HOME is not set. " "Set it to the async-profiler installation directory.", file=sys.stderr, ) sys.exit(1) lib_ext = "dylib" if sys.platform == "darwin" else "so" ap_lib = os.path.join(ap_home, "lib", f"libasyncProfiler.{lib_ext}") ap_dir = args.async_profiler_dir ap_event = args.async_profiler_event ap_fmt = args.async_profiler_format ext = {"flamegraph": "html", "jfr": "jfr", "collapsed": "txt", "text": "txt"}[ap_fmt] driver_ap = ( f"-agentpath:{ap_lib}=start,event={ap_event}," f"{ap_fmt},file={ap_dir}/driver.{ext}" ) executor_ap = ( f"-agentpath:{ap_lib}=start,event={ap_event}," f"{ap_fmt},file={ap_dir}/executor.{ext}" ) for spark_key, ap_opts in [ ("spark.driver.extraJavaOptions", driver_ap), ("spark.executor.extraJavaOptions", executor_ap), ]: existing = conf.get(spark_key, "") conf[spark_key] = f"{existing} {ap_opts}".strip() for key, val in sorted(conf.items()): cmd += ["--conf", f"{key}={val}"] # tpcbench.py path cmd.append("tpcbench.py") # tpcbench args engine_name = config.get("engine", {}).get("name", args.engine) cmd += ["--name", engine_name] cmd += ["--benchmark", benchmark] use_iceberg = config.get("tpcbench_args", {}).get("use_iceberg", False) if use_iceberg: cmd += ["--catalog", resolve_env("${ICEBERG_CATALOG}")] cmd += ["--database", resolve_env("${ICEBERG_DATABASE}")] else: data_var = profile["data_env"] data_val = os.environ.get(data_var, "") cmd += ["--data", data_val] cmd += ["--output", args.output] cmd += ["--iterations", str(args.iterations)] if args.query is not None: cmd += ["--query", str(args.query)] if profile["format"] and not use_iceberg: cmd += ["--format", profile["format"]] return cmd def restart_spark(): """Stop and start Spark master and worker.""" spark_home = os.environ["SPARK_HOME"] sbin = os.path.join(spark_home, "sbin") spark_master = os.environ["SPARK_MASTER"] # Stop (ignore errors) subprocess.run( [os.path.join(sbin, "stop-master.sh")], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) subprocess.run( [os.path.join(sbin, "stop-worker.sh")], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) # Start (check errors) r = subprocess.run([os.path.join(sbin, "start-master.sh")]) if r.returncode != 0: print("Error: Failed to start Spark master", file=sys.stderr) sys.exit(1) r = subprocess.run([os.path.join(sbin, "start-worker.sh"), spark_master]) if r.returncode != 0: print("Error: Failed to start Spark worker", file=sys.stderr) sys.exit(1) def main(): parser = argparse.ArgumentParser( description="Consolidated TPC benchmark runner for Spark-based engines." ) parser.add_argument( "--engine", required=True, help="Engine name (matches a TOML file in engines/)", ) parser.add_argument( "--benchmark", required=True, choices=["tpch", "tpcds"], help="Benchmark to run", ) parser.add_argument( "--iterations", type=int, default=1, help="Number of iterations (default: 1)" ) parser.add_argument( "--output", default=".", help="Output directory (default: .)" ) parser.add_argument( "--query", type=int, default=None, help="Run a single query number" ) parser.add_argument( "--no-restart", action="store_true", help="Skip Spark master/worker restart", ) parser.add_argument( "--dry-run", action="store_true", help="Print the spark-submit command without executing", ) parser.add_argument( "--jfr", action="store_true", help="Enable Java Flight Recorder profiling for driver and executors", ) parser.add_argument( "--jfr-dir", default="/results/jfr", help="Directory for JFR output files (default: /results/jfr)", ) parser.add_argument( "--async-profiler", action="store_true", help="Enable async-profiler for driver and executors (profiles Java + native code)", ) parser.add_argument( "--async-profiler-dir", default="/results/async-profiler", help="Directory for async-profiler output files (default: /results/async-profiler)", ) parser.add_argument( "--async-profiler-event", default="cpu", help="async-profiler event type: cpu, wall, alloc, lock, etc. (default: cpu)", ) parser.add_argument( "--async-profiler-format", default="flamegraph", choices=["flamegraph", "jfr", "collapsed", "text"], help="async-profiler output format (default: flamegraph)", ) args = parser.parse_args() config = load_engine_config(args.engine) # Apply env defaults and exports before validation apply_env_defaults(config) apply_env_exports(config) check_common_env() check_required_env(config) check_benchmark_env(config, args.benchmark) # Restart Spark unless --no-restart or --dry-run if not args.no_restart and not args.dry_run: restart_spark() # Create profiling output directories (skip for dry-run) if not args.dry_run: if args.jfr: os.makedirs(args.jfr_dir, exist_ok=True) if args.async_profiler: os.makedirs(args.async_profiler_dir, exist_ok=True) cmd = build_spark_submit_cmd(config, args.benchmark, args) if args.dry_run: # Group paired arguments (e.g. --conf key=value) on one line parts = [] i = 0 while i < len(cmd): token = cmd[i] if token.startswith("--") and i + 1 < len(cmd) and not cmd[i + 1].startswith("--"): parts.append(f"{token} {cmd[i + 1]}") i += 2 else: parts.append(token) i += 1 print(" \\\n ".join(parts)) else: r = subprocess.run(cmd) sys.exit(r.returncode) if __name__ == "__main__": main() ================================================ FILE: benchmarks/tpc/tpcbench.py ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. """ TPC-H / TPC-DS benchmark runner. Supports two data sources: - Files: use --data with --format (parquet, csv, json) and optional --options - Iceberg tables: use --catalog and --database to specify the catalog location """ import argparse from datetime import datetime import hashlib import json import os from pyspark.sql import SparkSession import time from typing import Dict def dedup_columns(df): """Rename duplicate column aliases: a, a, b, b -> a, a_1, b, b_1""" counts = {} new_cols = [] for c in df.columns: if c not in counts: counts[c] = 0 new_cols.append(c) else: counts[c] += 1 new_cols.append(f"{c}_{counts[c]}") return df.toDF(*new_cols) def result_hash(rows): """Compute a deterministic MD5 hash from collected rows.""" sorted_rows = sorted(rows, key=lambda r: str(r)) h = hashlib.md5() for row in sorted_rows: h.update(str(row).encode("utf-8")) return h.hexdigest() def main( benchmark: str, data_path: str, catalog: str, database: str, iterations: int, output: str, name: str, format: str, query_num: int = None, write_path: str = None, options: Dict[str, str] = None, ): if options is None: options = {} query_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "queries", benchmark ) spark = SparkSession.builder \ .appName(f"{name} benchmark derived from {benchmark}") \ .getOrCreate() # Define tables for each benchmark if benchmark == "tpch": num_queries = 22 table_names = [ "customer", "lineitem", "nation", "orders", "part", "partsupp", "region", "supplier" ] elif benchmark == "tpcds": num_queries = 99 table_names = [ "call_center", "catalog_page", "catalog_returns", "catalog_sales", "customer", "customer_address", "customer_demographics", "date_dim", "time_dim", "household_demographics", "income_band", "inventory", "item", "promotion", "reason", "ship_mode", "store", "store_returns", "store_sales", "warehouse", "web_page", "web_returns", "web_sales", "web_site" ] else: raise ValueError(f"Invalid benchmark: {benchmark}") # Register tables from either files or Iceberg catalog using_iceberg = catalog is not None for table in table_names: if using_iceberg: source = f"{catalog}.{database}.{table}" print(f"Registering table {table} from {source}") df = spark.table(source) else: # Support both "customer/" and "customer.parquet/" layouts source = f"{data_path}/{table}.{format}" if not os.path.exists(source): source = f"{data_path}/{table}" print(f"Registering table {table} from {source}") df = spark.read.format(format).options(**options).load(source) df.createOrReplaceTempView(table) conf_dict = {k: v for k, v in spark.sparkContext.getConf().getAll()} results = { 'engine': 'datafusion-comet', 'benchmark': benchmark, 'spark_conf': conf_dict, } if using_iceberg: results['catalog'] = catalog results['database'] = database else: results['data_path'] = data_path for iteration in range(iterations): print(f"\n{'='*60}") print(f"Starting iteration {iteration + 1} of {iterations}") print(f"{'='*60}") iter_start_time = time.time() # Determine which queries to run if query_num is not None: if query_num < 1 or query_num > num_queries: raise ValueError( f"Query number {query_num} out of range. " f"Valid: 1-{num_queries} for {benchmark}" ) queries_to_run = [query_num] else: queries_to_run = range(1, num_queries + 1) for query in queries_to_run: spark.sparkContext.setJobDescription(f"{benchmark} q{query}") path = f"{query_path}/q{query}.sql" print(f"\nRunning query {query} from {path}") with open(path, "r") as f: text = f.read() queries = text.split(";") start_time = time.time() for sql in queries: sql = sql.strip().replace("create view", "create temp view") if len(sql) > 0: print(f"Executing: {sql[:100]}...") df = spark.sql(sql) df.explain("formatted") if write_path is not None: if len(df.columns) > 0: output_path = f"{write_path}/q{query}" deduped = dedup_columns(df) deduped.orderBy(*deduped.columns).coalesce(1).write.mode("overwrite").parquet(output_path) print(f"Results written to {output_path}") else: rows = df.collect() row_count = len(rows) row_hash = result_hash(rows) print(f"Query {query} returned {row_count} rows, hash={row_hash}") end_time = time.time() elapsed = end_time - start_time print(f"Query {query} took {elapsed:.2f} seconds") query_result = results.setdefault(query, {"durations": []}) query_result["durations"].append(round(elapsed, 3)) if "row_count" not in query_result and not write_path: query_result["row_count"] = row_count query_result["result_hash"] = row_hash iter_end_time = time.time() print(f"\nIteration {iteration + 1} took {iter_end_time - iter_start_time:.2f} seconds") # Write results result_str = json.dumps(results, indent=4) current_time_millis = int(datetime.now().timestamp() * 1000) results_path = f"{output}/{name}-{benchmark}-{current_time_millis}.json" print(f"\nWriting results to {results_path}") with open(results_path, "w") as f: f.write(result_str) spark.stop() if __name__ == "__main__": parser = argparse.ArgumentParser( description="TPC-H/TPC-DS benchmark runner for files or Iceberg tables" ) parser.add_argument( "--benchmark", required=True, help="Benchmark to run (tpch or tpcds)" ) # Data source - mutually exclusive: either file path or Iceberg catalog source_group = parser.add_mutually_exclusive_group(required=True) source_group.add_argument( "--data", help="Path to data files" ) source_group.add_argument( "--catalog", help="Iceberg catalog name" ) # Options for file-based reading parser.add_argument( "--format", default="parquet", help="Input file format: parquet, csv, json (only used with --data)" ) parser.add_argument( "--options", type=json.loads, default={}, help='Spark reader options as JSON string, e.g., \'{"header": "true"}\' (only used with --data)' ) # Options for Iceberg parser.add_argument( "--database", default="tpch", help="Database containing TPC tables (only used with --catalog)" ) parser.add_argument( "--iterations", type=int, default=1, help="Number of iterations" ) parser.add_argument( "--output", required=True, help="Path to write results JSON" ) parser.add_argument( "--name", required=True, help="Prefix for result file" ) parser.add_argument( "--query", type=int, help="Specific query number (1-based). If omitted, run all." ) parser.add_argument( "--write", help="Path to save query results as Parquet" ) args = parser.parse_args() main( args.benchmark, args.data, args.catalog, args.database, args.iterations, args.output, args.name, args.format, args.query, args.write, args.options, ) ================================================ FILE: common/pom.xml ================================================ 4.0.0 org.apache.datafusion comet-parent-spark${spark.version.short}_${scala.binary.version} 0.15.0-SNAPSHOT ../pom.xml comet-common-spark${spark.version.short}_${scala.binary.version} comet-common false org.apache.spark spark-sql_${scala.binary.version} org.apache.parquet parquet-column org.apache.parquet parquet-hadoop org.apache.parquet parquet-format-structures org.apache.arrow arrow-vector org.apache.arrow arrow-memory-unsafe org.apache.arrow arrow-c-data org.scala-lang.modules scala-collection-compat_${scala.binary.version} junit junit test org.assertj assertj-core test io.github.git-commit-id git-commit-id-maven-plugin ${git-commit-id-maven-plugin.version} get-the-git-infos revision initialize true ${project.build.outputDirectory}/comet-git-info.properties full ^git.branch$ ^git.build.*$ ^git.commit.id.(abbrev|full)$ ^git.remote.*$ org.apache.maven.plugins maven-shade-plugin package shade true true false true org.apache.arrow:* *:* **/*.thrift git.properties log4j.properties log4j2.properties arrow-git.properties org.apache.arrow:arrow-vector codegen/** org.apache.arrow ${comet.shade.packageName}.arrow org/apache/arrow/c/jni/JniWrapper org/apache/arrow/c/jni/PrivateData org/apache/arrow/c/jni/CDataJniException org/apache/arrow/c/ArrayStreamExporter$ExportedArrayStreamPrivateData net.alchim31.maven scala-maven-plugin org.codehaus.mojo build-helper-maven-plugin add-shim-source generate-sources add-source src/main/${shims.majorVerSrc} src/main/${shims.minorVerSrc} ${project.basedir}/src/main/resources ${project.basedir}/../native/target/x86_64-apple-darwin/release libcomet.dylib org/apache/comet/darwin/x86_64 ${project.basedir}/../native/target/aarch64-apple-darwin/release libcomet.dylib org/apache/comet/darwin/aarch64 ${jni.dir} libcomet.dylib libcomet.so comet.dll org/apache/comet/${platform}/${arch} ================================================ FILE: common/src/main/java/org/apache/arrow/c/AbstractCometSchemaImporter.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.arrow.c; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.types.pojo.Field; import org.apache.comet.IcebergApi; /** This is a simple wrapper around SchemaImporter to make it accessible from Java Arrow. */ public abstract class AbstractCometSchemaImporter { private final BufferAllocator allocator; private final SchemaImporter importer; private final CDataDictionaryProvider provider = new CDataDictionaryProvider(); public AbstractCometSchemaImporter(BufferAllocator allocator) { this.allocator = allocator; this.importer = new SchemaImporter(allocator); } public BufferAllocator getAllocator() { return allocator; } public CDataDictionaryProvider getProvider() { return provider; } public Field importField(ArrowSchema schema) { try { return importer.importField(schema, provider); } finally { schema.release(); schema.close(); } } /** * Imports data from ArrowArray/ArrowSchema into a FieldVector. This is basically the same as Java * Arrow `Data.importVector`. `Data.importVector` initiates `SchemaImporter` internally which is * used to fill dictionary ids for dictionary encoded vectors. Every call to `importVector` will * begin with dictionary ids starting from 0. So, separate calls to `importVector` will overwrite * dictionary ids. To avoid this, we need to use the same `SchemaImporter` instance for all calls * to `importVector`. */ public FieldVector importVector(ArrowArray array, ArrowSchema schema) { Field field = importField(schema); FieldVector vector = field.createVector(allocator); Data.importIntoVector(allocator, array, vector, provider); return vector; } @IcebergApi public void close() { provider.close(); } } ================================================ FILE: common/src/main/java/org/apache/arrow/c/ArrowImporter.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.arrow.c; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.types.pojo.Field; /** * This class is used to import Arrow schema and array from native execution/shuffle. We cannot use * Arrow's Java API to import schema and array directly because Arrow's Java API `Data.importField` * initiates a new `SchemaImporter` for each field. Each `SchemaImporter` maintains an internal * dictionary id counter. So the dictionary ids for multiple dictionary columns will conflict with * each other and cause data corruption. */ public class ArrowImporter { private final SchemaImporter importer; private final BufferAllocator allocator; public ArrowImporter(BufferAllocator allocator) { this.allocator = allocator; this.importer = new SchemaImporter(allocator); } Field importField(ArrowSchema schema, CDataDictionaryProvider provider) { try { return importer.importField(schema, provider); } finally { schema.release(); schema.close(); } } public FieldVector importVector( ArrowArray array, ArrowSchema schema, CDataDictionaryProvider provider) { Field field = importField(schema, provider); FieldVector vector = field.createVector(allocator); ArrayImporter importer = new ArrayImporter(allocator, vector, provider); importer.importArray(array); return vector; } } ================================================ FILE: common/src/main/java/org/apache/comet/CometNativeException.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet; /** Parent class for all exceptions thrown from Comet native side. */ public class CometNativeException extends CometRuntimeException { public CometNativeException(String message) { super(message); } } ================================================ FILE: common/src/main/java/org/apache/comet/CometOutOfMemoryError.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet; /** OOM error specific for Comet memory management */ public class CometOutOfMemoryError extends OutOfMemoryError { public CometOutOfMemoryError(String msg) { super(msg); } } ================================================ FILE: common/src/main/java/org/apache/comet/CometRuntimeException.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet; /** The parent class for all Comet runtime exceptions */ public class CometRuntimeException extends RuntimeException { public CometRuntimeException(String message) { super(message); } public CometRuntimeException(String message, Throwable cause) { super(message, cause); } } ================================================ FILE: common/src/main/java/org/apache/comet/CometSchemaImporter.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet; import org.apache.arrow.c.*; import org.apache.arrow.memory.BufferAllocator; /** This is a simple wrapper around SchemaImporter to make it accessible from Java Arrow. */ @IcebergApi public class CometSchemaImporter extends AbstractCometSchemaImporter { @IcebergApi public CometSchemaImporter(BufferAllocator allocator) { super(allocator); } } ================================================ FILE: common/src/main/java/org/apache/comet/IcebergApi.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet; import java.lang.annotation.Documented; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; /** * Indicates that the annotated element is part of the public API used by Apache Iceberg. * *

This annotation marks classes, methods, constructors, and fields that form the contract * between Comet and Iceberg. Changes to these APIs may break Iceberg's Comet integration, so * contributors should exercise caution and consider backward compatibility when modifying annotated * elements. * *

The Iceberg integration uses Comet's native Parquet reader for accelerated vectorized reads. * See the contributor guide documentation for details on how Iceberg uses these APIs. * * @see Apache Iceberg */ @Documented @Retention(RetentionPolicy.RUNTIME) @Target({ElementType.TYPE, ElementType.METHOD, ElementType.CONSTRUCTOR, ElementType.FIELD}) public @interface IcebergApi {} ================================================ FILE: common/src/main/java/org/apache/comet/NativeBase.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet; import java.io.BufferedReader; import java.io.File; import java.io.FilenameFilter; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.nio.file.Files; import java.nio.file.StandardCopyOption; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.spark.sql.comet.util.Utils; import static org.apache.comet.Constants.LOG_CONF_NAME; import static org.apache.comet.Constants.LOG_CONF_PATH; import static org.apache.comet.Constants.LOG_LEVEL_ENV; /** Base class for JNI bindings. MUST be inherited by all classes that introduce JNI APIs. */ public abstract class NativeBase { static final String ARROW_UNSAFE_MEMORY_ACCESS = "arrow.enable_unsafe_memory_access"; static final String ARROW_NULL_CHECK_FOR_GET = "arrow.enable_null_check_for_get"; private static final Logger LOG = LoggerFactory.getLogger(NativeBase.class); private static final String NATIVE_LIB_NAME = "comet"; private static final String libraryToLoad = System.mapLibraryName(NATIVE_LIB_NAME); private static boolean loaded = false; private static volatile Throwable loadErr = null; private static final String searchPattern = "libcomet-"; static { try { load(); } catch (Throwable th) { LOG.warn("Failed to load comet library", th); // logging may not be initialized yet, so also write to stderr System.err.println("Failed to load comet library: " + th.getMessage()); loadErr = th; } } public static synchronized boolean isLoaded() throws Throwable { if (loadErr != null) { throw loadErr; } return loaded; } // Only for testing static synchronized void setLoaded(boolean b) { loaded = b; } static synchronized void load() { if (loaded) { return; } cleanupOldTempLibs(); // Check if the arch used by JDK is the same as arch on the host machine, in particular, // whether x86_64 JDK is used in arm64 Mac if (!checkArch()) { LOG.warn( "Comet is disabled. JDK compiled for x86_64 is used in a Mac based on Apple Silicon. " + "In order to use Comet, Please install a JDK version for ARM64 architecture"); return; } // Try to load Comet library from the java.library.path. try { System.loadLibrary(NATIVE_LIB_NAME); loaded = true; } catch (UnsatisfiedLinkError ex) { // Doesn't exist, so proceed to loading bundled library. bundleLoadLibrary(); } initWithLogConf(); // Only set the Arrow properties when debugging mode is off if (!(boolean) CometConf.COMET_DEBUG_ENABLED().get()) { setArrowProperties(); } } /** * Use the bundled native libraries. Functionally equivalent to System.loadLibrary. */ private static void bundleLoadLibrary() { String resourceName = resourceName(); InputStream is = NativeBase.class.getResourceAsStream(resourceName); if (is == null) { throw new UnsupportedOperationException( "Unsupported OS/arch, cannot find " + resourceName + ". Please try building from source."); } File tempLib = null; File tempLibLock = null; try { // Create the .lck file first to avoid a race condition // with other concurrently running Java processes using Comet. tempLibLock = File.createTempFile(searchPattern, "." + os().libExtension + ".lck"); tempLib = new File(tempLibLock.getAbsolutePath().replaceFirst(".lck$", "")); // copy to tempLib Files.copy(is, tempLib.toPath(), StandardCopyOption.REPLACE_EXISTING); System.load(tempLib.getAbsolutePath()); loaded = true; } catch (IOException e) { throw new IllegalStateException("Cannot unpack libcomet: " + e); } finally { if (!loaded) { if (tempLib != null && tempLib.exists()) { if (!tempLib.delete()) { LOG.error( "Cannot unpack libcomet / cannot delete a temporary native library " + tempLib); } } if (tempLibLock != null && tempLibLock.exists()) { if (!tempLibLock.delete()) { LOG.error( "Cannot unpack libcomet / cannot delete a temporary lock file " + tempLibLock); } } } else { tempLib.deleteOnExit(); tempLibLock.deleteOnExit(); } } } private static void initWithLogConf() { String logConfPath = System.getProperty(LOG_CONF_PATH(), Utils.getConfPath(LOG_CONF_NAME())); String logLevel = System.getenv(LOG_LEVEL_ENV()); // If both the system property and the environmental variable failed to find a log // configuration, then fall back to using the deployed default if (logConfPath == null) { LOG.info( "Couldn't locate log file from either COMET_CONF_DIR or comet.log.file.path. " + "Using default log configuration with {} log level which emits to stderr", logLevel == null ? "INFO" : logLevel); logConfPath = ""; } else { // Ignore log level if a log configuration file is specified if (logLevel != null) { LOG.warn("Ignoring log level {} because a log configuration file is specified", logLevel); } LOG.info("Using {} for native library logging", logConfPath); } init(logConfPath, logLevel); } private static void cleanupOldTempLibs() { String tempFolder = new File(System.getProperty("java.io.tmpdir")).getAbsolutePath(); File dir = new File(tempFolder); File[] tempLibFiles = dir.listFiles( new FilenameFilter() { public boolean accept(File dir, String name) { return name.startsWith(searchPattern) && !name.endsWith(".lck"); } }); if (tempLibFiles != null) { for (File tempLibFile : tempLibFiles) { File lckFile = new File(tempLibFile.getAbsolutePath() + ".lck"); if (!lckFile.exists()) { try { tempLibFile.delete(); } catch (SecurityException e) { LOG.error("Failed to delete old temp lib", e); } } } } } // Set Arrow related properties upon initializing native, such as enabling unsafe memory access // as well as disabling null check for get, for performance reasons. private static void setArrowProperties() { setPropertyIfNull(ARROW_UNSAFE_MEMORY_ACCESS, "true"); setPropertyIfNull(ARROW_NULL_CHECK_FOR_GET, "false"); } private static void setPropertyIfNull(String key, String value) { if (System.getProperty(key) == null) { LOG.info("Setting system property {} to {}", key, value); System.setProperty(key, value); } else { LOG.info( "Skip setting system property {} to {}, because it is already set to {}", key, value, System.getProperty(key)); } } private enum OS { // Even on Windows, the default compiler from cpptasks (gcc) uses .so as a shared lib extension WINDOWS("win32", "so"), LINUX("linux", "so"), MAC("darwin", "dylib"), SOLARIS("solaris", "so"); public final String name, libExtension; OS(String name, String libExtension) { this.name = name; this.libExtension = libExtension; } } private static String arch() { return System.getProperty("os.arch"); } private static OS os() { String osName = System.getProperty("os.name"); if (osName.contains("Linux")) { return OS.LINUX; } else if (osName.contains("Mac")) { return OS.MAC; } else if (osName.contains("Windows")) { return OS.WINDOWS; } else if (osName.contains("Solaris") || osName.contains("SunOS")) { return OS.SOLARIS; } else { throw new UnsupportedOperationException("Unsupported operating system: " + osName); } } // For some reason users will get JVM crash when running Comet that is compiled for `aarch64` // using a JVM that is compiled against `amd64`. Here we check if that is the case and fallback // to Spark accordingly. private static boolean checkArch() { if (os() == OS.MAC) { try { String javaArch = arch(); Process process = Runtime.getRuntime().exec("uname -a"); if (process.waitFor() == 0) { BufferedReader in = new BufferedReader(new InputStreamReader(process.getInputStream())); String line; while ((line = in.readLine()) != null) { if (javaArch.equals("x86_64") && line.contains("ARM64")) { return false; } } } } catch (IOException | InterruptedException e) { LOG.warn("Error parsing host architecture", e); } } return true; } private static String resourceName() { OS os = os(); String packagePrefix = NativeBase.class.getPackage().getName().replace('.', '/'); return "/" + packagePrefix + "/" + os.name + "/" + arch() + "/" + libraryToLoad; } /** * Initialize the native library through JNI. * * @param logConfPath location to the native log configuration file */ static native void init(String logConfPath, String logLevel); /** * Check if a specific feature is enabled in the native library. * * @param featureName The name of the feature to check (e.g., "hdfs", "jemalloc", "hdfs-opendal") * @return true if the feature is enabled, false otherwise */ public static native boolean isFeatureEnabled(String featureName); } ================================================ FILE: common/src/main/java/org/apache/comet/ParquetRuntimeException.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet; /** The parent class for the subset of Comet runtime exceptions related to Parquet. */ public class ParquetRuntimeException extends CometRuntimeException { public ParquetRuntimeException(String message) { super(message); } public ParquetRuntimeException(String message, Throwable cause) { super(message, cause); } } ================================================ FILE: common/src/main/java/org/apache/comet/exceptions/CometQueryExecutionException.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.exceptions; import org.apache.comet.CometNativeException; /** * Exception thrown from Comet native execution containing JSON-encoded error information. The * message contains a JSON object with the following structure: * * 

 * {
 *   "errorType": "DivideByZero",
 *   "errorClass": "DIVIDE_BY_ZERO",
 *   "params": { ... },
 *   "context": { "sqlText": "...", "startOffset": 0, "stopOffset": 10 },
 *   "hint": "Use `try_divide` to tolerate divisor being 0"
 * }
 *
* * CometExecIterator parses this JSON and converts it to the appropriate Spark exception by calling * the corresponding QueryExecutionErrors.* method. */ public final class CometQueryExecutionException extends CometNativeException { /** * Creates a new CometQueryExecutionException with a JSON-encoded error message. * * @param jsonMessage JSON string containing error information */ public CometQueryExecutionException(String jsonMessage) { super(jsonMessage); } /** * Returns true if the message appears to be JSON-formatted. This is used to distinguish between * JSON-encoded errors and legacy error messages. * * @return true if message starts with '{' and ends with '}' */ public boolean isJsonMessage() { String msg = getMessage(); if (msg == null) return false; String trimmed = msg.trim(); return trimmed.startsWith("{") && trimmed.endsWith("}"); } } ================================================ FILE: common/src/main/java/org/apache/comet/parquet/AbstractColumnReader.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.parquet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.schema.Type; import org.apache.spark.sql.types.DataType; import org.apache.spark.sql.types.TimestampNTZType$; import org.apache.comet.CometConf; import org.apache.comet.IcebergApi; import org.apache.comet.vector.CometVector; /** Base class for Comet Parquet column reader implementations. */ @IcebergApi public abstract class AbstractColumnReader implements AutoCloseable { protected static final Logger LOG = LoggerFactory.getLogger(AbstractColumnReader.class); /** The Spark data type. */ protected final DataType type; /** The Spark data type. */ protected final Type fieldType; /** Parquet column descriptor. */ protected final ColumnDescriptor descriptor; /** * Whether to always return 128 bit decimals, regardless of its precision. If false, this will * return 32, 64 or 128 bit decimals depending on the precision. */ protected final boolean useDecimal128; /** * Whether to return dates/timestamps that were written with legacy hybrid (Julian + Gregorian) * calendar as it is. If this is true, Comet will return them as it is, instead of rebasing them * to the new Proleptic Gregorian calendar. If this is false, Comet will throw exceptions when * seeing these dates/timestamps. */ protected final boolean useLegacyDateTimestamp; /** The size of one batch, gets updated by 'readBatch' */ protected int batchSize; /** A pointer to the native implementation of ColumnReader. */ @IcebergApi protected long nativeHandle; AbstractColumnReader( DataType type, Type fieldType, ColumnDescriptor descriptor, boolean useDecimal128, boolean useLegacyDateTimestamp) { this.type = type; this.fieldType = fieldType; this.descriptor = descriptor; this.useDecimal128 = useDecimal128; this.useLegacyDateTimestamp = useLegacyDateTimestamp; } AbstractColumnReader( DataType type, ColumnDescriptor descriptor, boolean useDecimal128, boolean useLegacyDateTimestamp) { this(type, null, descriptor, useDecimal128, useLegacyDateTimestamp); TypeUtil.checkParquetType(descriptor, type); } ColumnDescriptor getDescriptor() { return descriptor; } String getPath() { return String.join(".", this.descriptor.getPath()); } /** * Set the batch size of this reader to be 'batchSize'. Also initializes the native column reader. */ @IcebergApi public void setBatchSize(int batchSize) { assert nativeHandle == 0 : "Native column reader shouldn't be initialized before " + "'setBatchSize' is called"; this.batchSize = batchSize; initNative(); } /** * Reads a batch of 'total' new rows. * * @param total the total number of rows to read */ public abstract void readBatch(int total); /** Returns the {@link CometVector} read by this reader. */ public abstract CometVector currentBatch(); @IcebergApi @Override public void close() { if (nativeHandle != 0) { LOG.debug("Closing the column reader"); Native.closeColumnReader(nativeHandle); nativeHandle = 0; } } protected void initNative() { LOG.debug("initializing the native column reader"); DataType readType = (boolean) CometConf.COMET_SCHEMA_EVOLUTION_ENABLED().get() ? type : null; boolean useLegacyDateTimestampOrNTZ = useLegacyDateTimestamp || type == TimestampNTZType$.MODULE$; nativeHandle = Utils.initColumnReader( descriptor, readType, batchSize, useDecimal128, useLegacyDateTimestampOrNTZ); } } ================================================ FILE: common/src/main/java/org/apache/comet/parquet/ArrowConstantColumnReader.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.parquet; import java.math.BigDecimal; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.*; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.catalyst.util.ResolveDefaultColumns; import org.apache.spark.sql.types.*; import org.apache.spark.unsafe.types.UTF8String; import org.apache.comet.vector.CometPlainVector; import org.apache.comet.vector.CometVector; /** * A column reader that returns constant vectors using Arrow Java vectors directly (no native * mutable buffers). Used for partition columns and missing columns in the native_iceberg_compat * scan path. * *

The vector is filled with the constant value repeated for every row in the batch. This is * necessary because the underlying Arrow vector's buffers must be large enough to match the * reported value count — otherwise variable-width types (strings, binary) would have undersized * offset buffers, causing out-of-bounds reads on the native side. */ public class ArrowConstantColumnReader extends AbstractColumnReader { private final BufferAllocator allocator = new RootAllocator(); private boolean isNull; private Object value; private FieldVector fieldVector; private CometPlainVector vector; private int currentSize; /** Constructor for missing columns (default values from schema). */ ArrowConstantColumnReader(StructField field, int batchSize, boolean useDecimal128) { super(field.dataType(), TypeUtil.convertToParquet(field), useDecimal128, false); this.batchSize = batchSize; this.value = ResolveDefaultColumns.getExistenceDefaultValues(new StructType(new StructField[] {field}))[ 0]; initVector(value, batchSize); } /** Constructor for partition columns with values from a row. */ ArrowConstantColumnReader( StructField field, int batchSize, InternalRow values, int index, boolean useDecimal128) { super(field.dataType(), TypeUtil.convertToParquet(field), useDecimal128, false); this.batchSize = batchSize; Object v = values.get(index, field.dataType()); this.value = v; initVector(v, batchSize); } @Override public void setBatchSize(int batchSize) { close(); this.batchSize = batchSize; initVector(value, batchSize); } @Override public void readBatch(int total) { if (total != currentSize) { close(); initVector(value, total); } } @Override public CometVector currentBatch() { return vector; } @Override public void close() { if (vector != null) { vector.close(); vector = null; } if (fieldVector != null) { fieldVector.close(); fieldVector = null; } } private void initVector(Object value, int count) { currentSize = count; if (value == null) { isNull = true; fieldVector = createNullVector(count); } else { isNull = false; fieldVector = createFilledVector(value, count); } vector = new CometPlainVector(fieldVector, useDecimal128, false, true); } /** Creates a vector of the correct type with {@code count} null values. */ private FieldVector createNullVector(int count) { String name = "constant"; FieldVector v; if (type == DataTypes.BooleanType) { v = new BitVector(name, allocator); } else if (type == DataTypes.ByteType) { v = new TinyIntVector(name, allocator); } else if (type == DataTypes.ShortType) { v = new SmallIntVector(name, allocator); } else if (type == DataTypes.IntegerType || type == DataTypes.DateType) { v = new IntVector(name, allocator); } else if (type == DataTypes.LongType || type == DataTypes.TimestampType || type == TimestampNTZType$.MODULE$) { v = new BigIntVector(name, allocator); } else if (type == DataTypes.FloatType) { v = new Float4Vector(name, allocator); } else if (type == DataTypes.DoubleType) { v = new Float8Vector(name, allocator); } else if (type == DataTypes.BinaryType) { v = new VarBinaryVector(name, allocator); } else if (type == DataTypes.StringType) { v = new VarCharVector(name, allocator); } else if (type instanceof DecimalType) { DecimalType dt = (DecimalType) type; if (!useDecimal128 && dt.precision() <= Decimal.MAX_INT_DIGITS()) { v = new IntVector(name, allocator); } else if (!useDecimal128 && dt.precision() <= Decimal.MAX_LONG_DIGITS()) { v = new BigIntVector(name, allocator); } else { v = new DecimalVector(name, allocator, dt.precision(), dt.scale()); } } else { throw new UnsupportedOperationException("Unsupported Spark type: " + type); } v.setValueCount(count); return v; } /** Creates a vector filled with {@code count} copies of the given value. */ private FieldVector createFilledVector(Object value, int count) { String name = "constant"; if (type == DataTypes.BooleanType) { BitVector v = new BitVector(name, allocator); v.allocateNew(count); int bit = (boolean) value ? 1 : 0; for (int i = 0; i < count; i++) v.setSafe(i, bit); v.setValueCount(count); return v; } else if (type == DataTypes.ByteType) { TinyIntVector v = new TinyIntVector(name, allocator); v.allocateNew(count); byte val = (byte) value; for (int i = 0; i < count; i++) v.setSafe(i, val); v.setValueCount(count); return v; } else if (type == DataTypes.ShortType) { SmallIntVector v = new SmallIntVector(name, allocator); v.allocateNew(count); short val = (short) value; for (int i = 0; i < count; i++) v.setSafe(i, val); v.setValueCount(count); return v; } else if (type == DataTypes.IntegerType || type == DataTypes.DateType) { IntVector v = new IntVector(name, allocator); v.allocateNew(count); int val = (int) value; for (int i = 0; i < count; i++) v.setSafe(i, val); v.setValueCount(count); return v; } else if (type == DataTypes.LongType || type == DataTypes.TimestampType || type == TimestampNTZType$.MODULE$) { BigIntVector v = new BigIntVector(name, allocator); v.allocateNew(count); long val = (long) value; for (int i = 0; i < count; i++) v.setSafe(i, val); v.setValueCount(count); return v; } else if (type == DataTypes.FloatType) { Float4Vector v = new Float4Vector(name, allocator); v.allocateNew(count); float val = (float) value; for (int i = 0; i < count; i++) v.setSafe(i, val); v.setValueCount(count); return v; } else if (type == DataTypes.DoubleType) { Float8Vector v = new Float8Vector(name, allocator); v.allocateNew(count); double val = (double) value; for (int i = 0; i < count; i++) v.setSafe(i, val); v.setValueCount(count); return v; } else if (type == DataTypes.BinaryType) { VarBinaryVector v = new VarBinaryVector(name, allocator); v.allocateNew(count); byte[] bytes = (byte[]) value; for (int i = 0; i < count; i++) v.setSafe(i, bytes, 0, bytes.length); v.setValueCount(count); return v; } else if (type == DataTypes.StringType) { VarCharVector v = new VarCharVector(name, allocator); v.allocateNew(count); byte[] bytes = ((UTF8String) value).getBytes(); for (int i = 0; i < count; i++) v.setSafe(i, bytes, 0, bytes.length); v.setValueCount(count); return v; } else if (type instanceof DecimalType) { DecimalType dt = (DecimalType) type; Decimal d = (Decimal) value; if (!useDecimal128 && dt.precision() <= Decimal.MAX_INT_DIGITS()) { IntVector v = new IntVector(name, allocator); v.allocateNew(count); int val = (int) d.toUnscaledLong(); for (int i = 0; i < count; i++) v.setSafe(i, val); v.setValueCount(count); return v; } else if (!useDecimal128 && dt.precision() <= Decimal.MAX_LONG_DIGITS()) { BigIntVector v = new BigIntVector(name, allocator); v.allocateNew(count); long val = d.toUnscaledLong(); for (int i = 0; i < count; i++) v.setSafe(i, val); v.setValueCount(count); return v; } else { DecimalVector v = new DecimalVector(name, allocator, dt.precision(), dt.scale()); v.allocateNew(count); BigDecimal bd = d.toJavaBigDecimal(); for (int i = 0; i < count; i++) v.setSafe(i, bd); v.setValueCount(count); return v; } } else { throw new UnsupportedOperationException("Unsupported Spark type: " + type); } } } ================================================ FILE: common/src/main/java/org/apache/comet/parquet/ArrowRowIndexColumnReader.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.parquet; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.BigIntVector; import org.apache.spark.sql.types.*; import org.apache.comet.vector.CometPlainVector; import org.apache.comet.vector.CometVector; /** * A column reader that computes row indices in Java and creates Arrow BigIntVectors directly (no * native mutable buffers). Used for the row index metadata column in the native_iceberg_compat scan * path. * *

The {@code indices} array contains alternating pairs of (start_index, count) representing * ranges of sequential row indices within each row group. */ public class ArrowRowIndexColumnReader extends AbstractColumnReader { private final BufferAllocator allocator = new RootAllocator(); /** Alternating (start_index, count) pairs from row groups. */ private final long[] indices; /** Number of row indices consumed so far across batches. */ private long offset; private BigIntVector fieldVector; private CometPlainVector vector; public ArrowRowIndexColumnReader(StructField field, int batchSize, long[] indices) { super(field.dataType(), TypeUtil.convertToParquet(field), false, false); this.indices = indices; this.batchSize = batchSize; } @Override public void setBatchSize(int batchSize) { close(); this.batchSize = batchSize; } @Override public void readBatch(int total) { close(); fieldVector = new BigIntVector("row_index", allocator); fieldVector.allocateNew(total); // Port of Rust set_indices: iterate (start, count) pairs, skip offset rows, fill up to total. long skipped = 0; int filled = 0; for (int i = 0; i < indices.length && filled < total; i += 2) { long index = indices[i]; long count = indices[i + 1]; long skip = Math.min(count, offset - skipped); skipped += skip; if (count == skip) { continue; } long remaining = Math.min(count - skip, total - filled); for (long j = 0; j < remaining; j++) { fieldVector.setSafe(filled, index + skip + j); filled++; } } offset += filled; fieldVector.setValueCount(filled); vector = new CometPlainVector(fieldVector, false, false, false); vector.setNumValues(filled); } @Override public CometVector currentBatch() { return vector; } @Override public void close() { if (vector != null) { vector.close(); vector = null; } if (fieldVector != null) { fieldVector.close(); fieldVector = null; } } } ================================================ FILE: common/src/main/java/org/apache/comet/parquet/BloomFilterReader.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.parquet; import java.io.IOException; import java.util.HashMap; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.parquet.column.values.bloomfilter.BlockSplitBloomFilter; import org.apache.parquet.column.values.bloomfilter.BloomFilter; import org.apache.parquet.crypto.AesCipher; import org.apache.parquet.crypto.InternalColumnDecryptionSetup; import org.apache.parquet.crypto.InternalFileDecryptor; import org.apache.parquet.crypto.ModuleCipherFactory; import org.apache.parquet.crypto.ParquetCryptoRuntimeException; import org.apache.parquet.filter2.predicate.FilterPredicate; import org.apache.parquet.filter2.predicate.Operators; import org.apache.parquet.filter2.predicate.UserDefinedPredicate; import org.apache.parquet.format.BlockCipher; import org.apache.parquet.format.BloomFilterHeader; import org.apache.parquet.format.Util; import org.apache.parquet.hadoop.metadata.BlockMetaData; import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; import org.apache.parquet.hadoop.metadata.ColumnPath; import org.apache.parquet.io.SeekableInputStream; public class BloomFilterReader implements FilterPredicate.Visitor { private static final Logger LOG = LoggerFactory.getLogger(BloomFilterReader.class); private static final boolean BLOCK_MIGHT_MATCH = false; private static final boolean BLOCK_CANNOT_MATCH = true; private final Map columns; private final Map cache = new HashMap<>(); private final InternalFileDecryptor fileDecryptor; private final SeekableInputStream inputStream; BloomFilterReader( BlockMetaData block, InternalFileDecryptor fileDecryptor, SeekableInputStream inputStream) { this.columns = new HashMap<>(); for (ColumnChunkMetaData column : block.getColumns()) { columns.put(column.getPath(), column); } this.fileDecryptor = fileDecryptor; this.inputStream = inputStream; } @Override public > Boolean visit(Operators.Eq eq) { T value = eq.getValue(); if (value == null) { // the bloom filter bitset contains only non-null values so isn't helpful. this // could check the column stats, but the StatisticsFilter is responsible return BLOCK_MIGHT_MATCH; } Operators.Column filterColumn = eq.getColumn(); ColumnChunkMetaData meta = columns.get(filterColumn.getColumnPath()); if (meta == null) { // the column isn't in this file so all values are null, but the value // must be non-null because of the above check. return BLOCK_CANNOT_MATCH; } try { BloomFilter bloomFilter = readBloomFilter(meta); if (bloomFilter != null && !bloomFilter.findHash(bloomFilter.hash(value))) { return BLOCK_CANNOT_MATCH; } } catch (RuntimeException e) { LOG.warn(e.getMessage()); return BLOCK_MIGHT_MATCH; } return BLOCK_MIGHT_MATCH; } @Override public > Boolean visit(Operators.NotEq notEq) { return BLOCK_MIGHT_MATCH; } @Override public > Boolean visit(Operators.Lt lt) { return BLOCK_MIGHT_MATCH; } @Override public > Boolean visit(Operators.LtEq ltEq) { return BLOCK_MIGHT_MATCH; } @Override public > Boolean visit(Operators.Gt gt) { return BLOCK_MIGHT_MATCH; } @Override public > Boolean visit(Operators.GtEq gtEq) { return BLOCK_MIGHT_MATCH; } @Override public Boolean visit(Operators.And and) { return and.getLeft().accept(this) || and.getRight().accept(this); } @Override public Boolean visit(Operators.Or or) { return or.getLeft().accept(this) && or.getRight().accept(this); } @Override public Boolean visit(Operators.Not not) { throw new IllegalArgumentException( "This predicate " + not + " contains a not! Did you forget" + " to run this predicate through LogicalInverseRewriter?"); } @Override public , U extends UserDefinedPredicate> Boolean visit( Operators.UserDefined udp) { return visit(udp, false); } @Override public , U extends UserDefinedPredicate> Boolean visit( Operators.LogicalNotUserDefined udp) { return visit(udp.getUserDefined(), true); } private , U extends UserDefinedPredicate> Boolean visit( Operators.UserDefined ud, boolean inverted) { return BLOCK_MIGHT_MATCH; } BloomFilter readBloomFilter(ColumnChunkMetaData meta) { if (cache.containsKey(meta.getPath())) { return cache.get(meta.getPath()); } try { if (!cache.containsKey(meta.getPath())) { BloomFilter bloomFilter = readBloomFilterInternal(meta); if (bloomFilter == null) { return null; } cache.put(meta.getPath(), bloomFilter); } return cache.get(meta.getPath()); } catch (IOException e) { LOG.error("Failed to read Bloom filter data", e); } return null; } private BloomFilter readBloomFilterInternal(ColumnChunkMetaData meta) throws IOException { long bloomFilterOffset = meta.getBloomFilterOffset(); if (bloomFilterOffset < 0) { return null; } // Prepare to decrypt Bloom filter (for encrypted columns) BlockCipher.Decryptor bloomFilterDecryptor = null; byte[] bloomFilterHeaderAAD = null; byte[] bloomFilterBitsetAAD = null; if (null != fileDecryptor && !fileDecryptor.plaintextFile()) { InternalColumnDecryptionSetup columnDecryptionSetup = fileDecryptor.getColumnSetup(meta.getPath()); if (columnDecryptionSetup.isEncrypted()) { bloomFilterDecryptor = columnDecryptionSetup.getMetaDataDecryptor(); bloomFilterHeaderAAD = AesCipher.createModuleAAD( fileDecryptor.getFileAAD(), ModuleCipherFactory.ModuleType.BloomFilterHeader, meta.getRowGroupOrdinal(), columnDecryptionSetup.getOrdinal(), -1); bloomFilterBitsetAAD = AesCipher.createModuleAAD( fileDecryptor.getFileAAD(), ModuleCipherFactory.ModuleType.BloomFilterBitset, meta.getRowGroupOrdinal(), columnDecryptionSetup.getOrdinal(), -1); } } // Read Bloom filter data header. inputStream.seek(bloomFilterOffset); BloomFilterHeader bloomFilterHeader; try { bloomFilterHeader = Util.readBloomFilterHeader(inputStream, bloomFilterDecryptor, bloomFilterHeaderAAD); } catch (IOException e) { LOG.warn("read no bloom filter"); return null; } int numBytes = bloomFilterHeader.getNumBytes(); if (numBytes <= 0 || numBytes > BlockSplitBloomFilter.UPPER_BOUND_BYTES) { LOG.warn("the read bloom filter size is wrong, size is {}", bloomFilterHeader.getNumBytes()); return null; } if (!bloomFilterHeader.getHash().isSetXXHASH() || !bloomFilterHeader.getAlgorithm().isSetBLOCK() || !bloomFilterHeader.getCompression().isSetUNCOMPRESSED()) { LOG.warn( "the read bloom filter is not supported yet, algorithm = {}, hash = {}, " + "compression = {}", bloomFilterHeader.getAlgorithm(), bloomFilterHeader.getHash(), bloomFilterHeader.getCompression()); return null; } byte[] bitset; if (null == bloomFilterDecryptor) { bitset = new byte[numBytes]; inputStream.readFully(bitset); } else { bitset = bloomFilterDecryptor.decrypt(inputStream, bloomFilterBitsetAAD); if (bitset.length != numBytes) { throw new ParquetCryptoRuntimeException("Wrong length of decrypted bloom filter bitset"); } } return new BlockSplitBloomFilter(bitset); } } ================================================ FILE: common/src/main/java/org/apache/comet/parquet/ColumnIndexReader.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.parquet; import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.parquet.crypto.AesCipher; import org.apache.parquet.crypto.InternalColumnDecryptionSetup; import org.apache.parquet.crypto.InternalFileDecryptor; import org.apache.parquet.crypto.ModuleCipherFactory; import org.apache.parquet.format.BlockCipher; import org.apache.parquet.format.Util; import org.apache.parquet.format.converter.ParquetMetadataConverter; import org.apache.parquet.hadoop.metadata.BlockMetaData; import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; import org.apache.parquet.hadoop.metadata.ColumnPath; import org.apache.parquet.internal.column.columnindex.ColumnIndex; import org.apache.parquet.internal.column.columnindex.OffsetIndex; import org.apache.parquet.internal.filter2.columnindex.ColumnIndexStore; import org.apache.parquet.internal.hadoop.metadata.IndexReference; import org.apache.parquet.io.SeekableInputStream; class ColumnIndexReader implements ColumnIndexStore { private static final Logger LOG = LoggerFactory.getLogger(ColumnIndexReader.class); // Used for columns are not in this parquet file private static final IndexStore MISSING_INDEX_STORE = new IndexStore() { @Override public ColumnIndex getColumnIndex() { return null; } @Override public OffsetIndex getOffsetIndex() { return null; } }; private static final ColumnIndexReader EMPTY = new ColumnIndexReader(new BlockMetaData(), Collections.emptySet(), null, null) { @Override public ColumnIndex getColumnIndex(ColumnPath column) { return null; } @Override public OffsetIndex getOffsetIndex(ColumnPath column) { throw new MissingOffsetIndexException(column); } }; private final InternalFileDecryptor fileDecryptor; private final SeekableInputStream inputStream; private final Map store; /** * Creates a column index store which lazily reads column/offset indexes for the columns in paths. * Paths are the set of columns used for the projection. */ static ColumnIndexReader create( BlockMetaData block, Set paths, InternalFileDecryptor fileDecryptor, SeekableInputStream inputStream) { try { return new ColumnIndexReader(block, paths, fileDecryptor, inputStream); } catch (MissingOffsetIndexException e) { return EMPTY; } } private ColumnIndexReader( BlockMetaData block, Set paths, InternalFileDecryptor fileDecryptor, SeekableInputStream inputStream) { this.fileDecryptor = fileDecryptor; this.inputStream = inputStream; Map store = new HashMap<>(); for (ColumnChunkMetaData column : block.getColumns()) { ColumnPath path = column.getPath(); if (paths.contains(path)) { store.put(path, new IndexStoreImpl(column)); } } this.store = store; } @Override public ColumnIndex getColumnIndex(ColumnPath column) { return store.getOrDefault(column, MISSING_INDEX_STORE).getColumnIndex(); } @Override public OffsetIndex getOffsetIndex(ColumnPath column) { return store.getOrDefault(column, MISSING_INDEX_STORE).getOffsetIndex(); } private interface IndexStore { ColumnIndex getColumnIndex(); OffsetIndex getOffsetIndex(); } private class IndexStoreImpl implements IndexStore { private final ColumnChunkMetaData meta; private ColumnIndex columnIndex; private boolean columnIndexRead; private final OffsetIndex offsetIndex; IndexStoreImpl(ColumnChunkMetaData meta) { this.meta = meta; OffsetIndex oi; try { oi = readOffsetIndex(meta); } catch (IOException e) { // If the I/O issue still stands it will fail the reading later; // otherwise we fail the filtering only with a missing offset index. LOG.warn("Unable to read offset index for column {}", meta.getPath(), e); oi = null; } if (oi == null) { throw new MissingOffsetIndexException(meta.getPath()); } offsetIndex = oi; } @Override public ColumnIndex getColumnIndex() { if (!columnIndexRead) { try { columnIndex = readColumnIndex(meta); } catch (IOException e) { // If the I/O issue still stands it will fail the reading later; // otherwise we fail the filtering only with a missing column index. LOG.warn("Unable to read column index for column {}", meta.getPath(), e); } columnIndexRead = true; } return columnIndex; } @Override public OffsetIndex getOffsetIndex() { return offsetIndex; } } // Visible for testing ColumnIndex readColumnIndex(ColumnChunkMetaData column) throws IOException { IndexReference ref = column.getColumnIndexReference(); if (ref == null) { return null; } inputStream.seek(ref.getOffset()); BlockCipher.Decryptor columnIndexDecryptor = null; byte[] columnIndexAAD = null; if (null != fileDecryptor && !fileDecryptor.plaintextFile()) { InternalColumnDecryptionSetup columnDecryptionSetup = fileDecryptor.getColumnSetup(column.getPath()); if (columnDecryptionSetup.isEncrypted()) { columnIndexDecryptor = columnDecryptionSetup.getMetaDataDecryptor(); columnIndexAAD = AesCipher.createModuleAAD( fileDecryptor.getFileAAD(), ModuleCipherFactory.ModuleType.ColumnIndex, column.getRowGroupOrdinal(), columnDecryptionSetup.getOrdinal(), -1); } } return ParquetMetadataConverter.fromParquetColumnIndex( column.getPrimitiveType(), Util.readColumnIndex(inputStream, columnIndexDecryptor, columnIndexAAD)); } // Visible for testing OffsetIndex readOffsetIndex(ColumnChunkMetaData column) throws IOException { IndexReference ref = column.getOffsetIndexReference(); if (ref == null) { return null; } inputStream.seek(ref.getOffset()); BlockCipher.Decryptor offsetIndexDecryptor = null; byte[] offsetIndexAAD = null; if (null != fileDecryptor && !fileDecryptor.plaintextFile()) { InternalColumnDecryptionSetup columnDecryptionSetup = fileDecryptor.getColumnSetup(column.getPath()); if (columnDecryptionSetup.isEncrypted()) { offsetIndexDecryptor = columnDecryptionSetup.getMetaDataDecryptor(); offsetIndexAAD = AesCipher.createModuleAAD( fileDecryptor.getFileAAD(), ModuleCipherFactory.ModuleType.OffsetIndex, column.getRowGroupOrdinal(), columnDecryptionSetup.getOrdinal(), -1); } } return ParquetMetadataConverter.fromParquetOffsetIndex( Util.readOffsetIndex(inputStream, offsetIndexDecryptor, offsetIndexAAD)); } } ================================================ FILE: common/src/main/java/org/apache/comet/parquet/ColumnPageReader.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.parquet; import java.io.IOException; import java.util.ArrayDeque; import java.util.List; import java.util.Queue; import org.apache.parquet.bytes.BytesInput; import org.apache.parquet.column.page.DataPage; import org.apache.parquet.column.page.DataPageV1; import org.apache.parquet.column.page.DataPageV2; import org.apache.parquet.column.page.DictionaryPage; import org.apache.parquet.column.page.PageReader; import org.apache.parquet.compression.CompressionCodecFactory; import org.apache.parquet.crypto.AesCipher; import org.apache.parquet.crypto.ModuleCipherFactory; import org.apache.parquet.format.BlockCipher; import org.apache.parquet.internal.column.columnindex.OffsetIndex; import org.apache.parquet.io.ParquetDecodingException; public class ColumnPageReader implements PageReader { private final CompressionCodecFactory.BytesInputDecompressor decompressor; private final long valueCount; private final Queue compressedPages; private final DictionaryPage compressedDictionaryPage; private final OffsetIndex offsetIndex; private final long rowCount; private int pageIndex = 0; private final BlockCipher.Decryptor blockDecryptor; private final byte[] dataPageAAD; private final byte[] dictionaryPageAAD; ColumnPageReader( CompressionCodecFactory.BytesInputDecompressor decompressor, List compressedPages, DictionaryPage compressedDictionaryPage, OffsetIndex offsetIndex, long rowCount, BlockCipher.Decryptor blockDecryptor, byte[] fileAAD, int rowGroupOrdinal, int columnOrdinal) { this.decompressor = decompressor; this.compressedPages = new ArrayDeque<>(compressedPages); this.compressedDictionaryPage = compressedDictionaryPage; long count = 0; for (DataPage p : compressedPages) { count += p.getValueCount(); } this.valueCount = count; this.offsetIndex = offsetIndex; this.rowCount = rowCount; this.blockDecryptor = blockDecryptor; if (blockDecryptor != null) { dataPageAAD = AesCipher.createModuleAAD( fileAAD, ModuleCipherFactory.ModuleType.DataPage, rowGroupOrdinal, columnOrdinal, 0); dictionaryPageAAD = AesCipher.createModuleAAD( fileAAD, ModuleCipherFactory.ModuleType.DictionaryPage, rowGroupOrdinal, columnOrdinal, -1); } else { dataPageAAD = null; dictionaryPageAAD = null; } } @Override public long getTotalValueCount() { return valueCount; } /** Returns the total value count of the current page. */ public int getPageValueCount() { return compressedPages.element().getValueCount(); } /** Skips the current page so it won't be returned by {@link #readPage()} */ public void skipPage() { compressedPages.poll(); pageIndex++; } @Override public DataPage readPage() { final DataPage compressedPage = compressedPages.poll(); if (compressedPage == null) { return null; } final int currentPageIndex = pageIndex++; if (null != blockDecryptor) { AesCipher.quickUpdatePageAAD(dataPageAAD, getPageOrdinal(currentPageIndex)); } return compressedPage.accept( new DataPage.Visitor() { @Override public DataPage visit(DataPageV1 dataPageV1) { try { BytesInput bytes = dataPageV1.getBytes(); if (null != blockDecryptor) { bytes = BytesInput.from(blockDecryptor.decrypt(bytes.toByteArray(), dataPageAAD)); } BytesInput decompressed = decompressor.decompress(bytes, dataPageV1.getUncompressedSize()); final DataPageV1 decompressedPage; if (offsetIndex == null) { decompressedPage = new DataPageV1( decompressed, dataPageV1.getValueCount(), dataPageV1.getUncompressedSize(), dataPageV1.getStatistics(), dataPageV1.getRlEncoding(), dataPageV1.getDlEncoding(), dataPageV1.getValueEncoding()); } else { long firstRowIndex = offsetIndex.getFirstRowIndex(currentPageIndex); decompressedPage = new DataPageV1( decompressed, dataPageV1.getValueCount(), dataPageV1.getUncompressedSize(), firstRowIndex, Math.toIntExact( offsetIndex.getLastRowIndex(currentPageIndex, rowCount) - firstRowIndex + 1), dataPageV1.getStatistics(), dataPageV1.getRlEncoding(), dataPageV1.getDlEncoding(), dataPageV1.getValueEncoding()); } if (dataPageV1.getCrc().isPresent()) { decompressedPage.setCrc(dataPageV1.getCrc().getAsInt()); } return decompressedPage; } catch (IOException e) { throw new ParquetDecodingException("could not decompress page", e); } } @Override public DataPage visit(DataPageV2 dataPageV2) { if (!dataPageV2.isCompressed() && offsetIndex == null && null == blockDecryptor) { return dataPageV2; } BytesInput pageBytes = dataPageV2.getData(); if (null != blockDecryptor) { try { pageBytes = BytesInput.from(blockDecryptor.decrypt(pageBytes.toByteArray(), dataPageAAD)); } catch (IOException e) { throw new ParquetDecodingException( "could not convert page ByteInput to byte array", e); } } if (dataPageV2.isCompressed()) { int uncompressedSize = Math.toIntExact( dataPageV2.getUncompressedSize() - dataPageV2.getDefinitionLevels().size() - dataPageV2.getRepetitionLevels().size()); try { pageBytes = decompressor.decompress(pageBytes, uncompressedSize); } catch (IOException e) { throw new ParquetDecodingException("could not decompress page", e); } } if (offsetIndex == null) { return DataPageV2.uncompressed( dataPageV2.getRowCount(), dataPageV2.getNullCount(), dataPageV2.getValueCount(), dataPageV2.getRepetitionLevels(), dataPageV2.getDefinitionLevels(), dataPageV2.getDataEncoding(), pageBytes, dataPageV2.getStatistics()); } else { return DataPageV2.uncompressed( dataPageV2.getRowCount(), dataPageV2.getNullCount(), dataPageV2.getValueCount(), offsetIndex.getFirstRowIndex(currentPageIndex), dataPageV2.getRepetitionLevels(), dataPageV2.getDefinitionLevels(), dataPageV2.getDataEncoding(), pageBytes, dataPageV2.getStatistics()); } } }); } @Override public DictionaryPage readDictionaryPage() { if (compressedDictionaryPage == null) { return null; } try { BytesInput bytes = compressedDictionaryPage.getBytes(); if (null != blockDecryptor) { bytes = BytesInput.from(blockDecryptor.decrypt(bytes.toByteArray(), dictionaryPageAAD)); } DictionaryPage decompressedPage = new DictionaryPage( decompressor.decompress(bytes, compressedDictionaryPage.getUncompressedSize()), compressedDictionaryPage.getDictionarySize(), compressedDictionaryPage.getEncoding()); if (compressedDictionaryPage.getCrc().isPresent()) { decompressedPage.setCrc(compressedDictionaryPage.getCrc().getAsInt()); } return decompressedPage; } catch (IOException e) { throw new ParquetDecodingException("Could not decompress dictionary page", e); } } private int getPageOrdinal(int currentPageIndex) { return offsetIndex == null ? currentPageIndex : offsetIndex.getPageOrdinal(currentPageIndex); } } ================================================ FILE: common/src/main/java/org/apache/comet/parquet/ColumnReader.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.parquet; import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.arrow.c.ArrowArray; import org.apache.arrow.c.ArrowSchema; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.dictionary.Dictionary; import org.apache.arrow.vector.types.pojo.DictionaryEncoding; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.Encoding; import org.apache.parquet.column.page.DataPage; import org.apache.parquet.column.page.DataPageV1; import org.apache.parquet.column.page.DataPageV2; import org.apache.parquet.column.page.DictionaryPage; import org.apache.parquet.column.page.PageReader; import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.spark.sql.types.DataType; import org.apache.comet.CometSchemaImporter; import org.apache.comet.IcebergApi; import org.apache.comet.vector.CometDecodedVector; import org.apache.comet.vector.CometDictionary; import org.apache.comet.vector.CometDictionaryVector; import org.apache.comet.vector.CometPlainVector; import org.apache.comet.vector.CometVector; @IcebergApi public class ColumnReader extends AbstractColumnReader { protected static final Logger LOG = LoggerFactory.getLogger(ColumnReader.class); protected final BufferAllocator ALLOCATOR = new RootAllocator(); /** * The current Comet vector holding all the values read by this column reader. Owned by this * reader and MUST be closed after use. */ private CometDecodedVector currentVector; /** Dictionary values for this column. Only set if the column is using dictionary encoding. */ protected CometDictionary dictionary; /** Reader for dictionary & data pages in the current column chunk. */ protected PageReader pageReader; /** Whether the first data page has been loaded. */ private boolean firstPageLoaded = false; /** * The number of nulls in the current batch, used when we are skipping importing of Arrow vectors, * in which case we'll simply update the null count of the existing vectors. */ int currentNumNulls; /** * The number of values in the current batch, used when we are skipping importing of Arrow * vectors, in which case we'll simply update the null count of the existing vectors. */ int currentNumValues; /** * Whether the last loaded vector contains any null value. This is used to determine if we can * skip vector reloading. If the flag is false, Arrow C API will skip to import the validity * buffer, and therefore we cannot skip vector reloading. */ boolean hadNull; private final CometSchemaImporter importer; private ArrowArray array = null; private ArrowSchema schema = null; ColumnReader( DataType type, ColumnDescriptor descriptor, CometSchemaImporter importer, int batchSize, boolean useDecimal128, boolean useLegacyDateTimestamp) { super(type, descriptor, useDecimal128, useLegacyDateTimestamp); assert batchSize > 0 : "Batch size must be positive, found " + batchSize; this.batchSize = batchSize; this.importer = importer; initNative(); } /** * Set the page reader for a new column chunk to read. Expects to call `readBatch` after this. * * @param pageReader the page reader for the new column chunk * @see Comet Issue #2079 */ @IcebergApi public void setPageReader(PageReader pageReader) throws IOException { this.pageReader = pageReader; DictionaryPage dictionaryPage = pageReader.readDictionaryPage(); if (dictionaryPage != null) { LOG.debug("dictionary page encoding = {}", dictionaryPage.getEncoding()); Native.setDictionaryPage( nativeHandle, dictionaryPage.getDictionarySize(), dictionaryPage.getBytes().toByteArray(), dictionaryPage.getEncoding().ordinal()); } } /** This method is called from Apache Iceberg. */ @IcebergApi public void setRowGroupReader(RowGroupReader rowGroupReader, ParquetColumnSpec columnSpec) throws IOException { ColumnDescriptor descriptor = Utils.buildColumnDescriptor(columnSpec); setPageReader(rowGroupReader.getPageReader(descriptor)); } @Override public void readBatch(int total) { LOG.debug("Start to batch of size = " + total); if (!firstPageLoaded) { readPage(); firstPageLoaded = true; } // Now first reset the current columnar batch so that it can be used to fill in a new batch // of values. Then, keep reading more data pages (via 'readBatch') until the current batch is // full, or we have read 'total' number of values. Native.resetBatch(nativeHandle); int left = total, nullsRead = 0; while (left > 0) { int[] array = Native.readBatch(nativeHandle, left); int valuesRead = array[0]; nullsRead += array[1]; if (valuesRead < left) { readPage(); } left -= valuesRead; } this.currentNumValues = total; this.currentNumNulls = nullsRead; } /** Returns the {@link CometVector} read by this reader. */ @Override public CometVector currentBatch() { return loadVector(); } @Override public void close() { if (currentVector != null) { currentVector.close(); currentVector = null; } super.close(); } /** Returns a decoded {@link CometDecodedVector Comet vector}. */ public CometDecodedVector loadVector() { LOG.debug("Reloading vector"); // Close the previous vector first to release struct memory allocated to import Arrow array & // schema from native side, through the C data interface if (currentVector != null) { currentVector.close(); } LogicalTypeAnnotation logicalTypeAnnotation = descriptor.getPrimitiveType().getLogicalTypeAnnotation(); boolean isUuid = logicalTypeAnnotation instanceof LogicalTypeAnnotation.UUIDLogicalTypeAnnotation; array = ArrowArray.allocateNew(ALLOCATOR); schema = ArrowSchema.allocateNew(ALLOCATOR); long arrayAddr = array.memoryAddress(); long schemaAddr = schema.memoryAddress(); Native.currentBatch(nativeHandle, arrayAddr, schemaAddr); FieldVector vector = importer.importVector(array, schema); DictionaryEncoding dictionaryEncoding = vector.getField().getDictionary(); CometPlainVector cometVector = new CometPlainVector(vector, useDecimal128); // Update whether the current vector contains any null values. This is used in the following // batch(s) to determine whether we can skip loading the native vector. hadNull = cometVector.hasNull(); if (dictionaryEncoding == null) { if (dictionary != null) { // This means the column was using dictionary encoding but now has fall-back to plain // encoding, on the native side. Setting 'dictionary' to null here, so we can use it as // a condition to check if we can re-use vector later. dictionary = null; } // Either the column is not dictionary encoded, or it was using dictionary encoding but // a new data page has switched back to use plain encoding. For both cases we should // return plain vector. currentVector = cometVector; return currentVector; } // We should already re-initiate `CometDictionary` here because `Data.importVector` API will // release the previous dictionary vector and create a new one. Dictionary arrowDictionary = importer.getProvider().lookup(dictionaryEncoding.getId()); CometPlainVector dictionaryVector = new CometPlainVector(arrowDictionary.getVector(), useDecimal128, isUuid); if (dictionary != null) { dictionary.setDictionaryVector(dictionaryVector); } else { dictionary = new CometDictionary(dictionaryVector); } currentVector = new CometDictionaryVector( cometVector, dictionary, importer.getProvider(), useDecimal128, false, isUuid); currentVector = new CometDictionaryVector(cometVector, dictionary, importer.getProvider(), useDecimal128); return currentVector; } protected void readPage() { DataPage page = pageReader.readPage(); if (page == null) { throw new RuntimeException("overreading: returned DataPage is null"); } ; int pageValueCount = page.getValueCount(); page.accept( new DataPage.Visitor() { @Override public Void visit(DataPageV1 dataPageV1) { LOG.debug("data page encoding = {}", dataPageV1.getValueEncoding()); if (dataPageV1.getDlEncoding() != Encoding.RLE && descriptor.getMaxDefinitionLevel() != 0) { throw new UnsupportedOperationException( "Unsupported encoding: " + dataPageV1.getDlEncoding()); } if (!isValidValueEncoding(dataPageV1.getValueEncoding())) { throw new UnsupportedOperationException( "Unsupported value encoding: " + dataPageV1.getValueEncoding()); } try { byte[] array = dataPageV1.getBytes().toByteArray(); Native.setPageV1( nativeHandle, pageValueCount, array, dataPageV1.getValueEncoding().ordinal()); } catch (IOException e) { throw new RuntimeException(e); } return null; } @Override public Void visit(DataPageV2 dataPageV2) { if (!isValidValueEncoding(dataPageV2.getDataEncoding())) { throw new UnsupportedOperationException( "Unsupported encoding: " + dataPageV2.getDataEncoding()); } try { Native.setPageV2( nativeHandle, pageValueCount, dataPageV2.getDefinitionLevels().toByteArray(), dataPageV2.getRepetitionLevels().toByteArray(), dataPageV2.getData().toByteArray(), dataPageV2.getDataEncoding().ordinal()); } catch (IOException e) { throw new RuntimeException(e); } return null; } }); } @SuppressWarnings("deprecation") private boolean isValidValueEncoding(Encoding encoding) { switch (encoding) { case PLAIN: case RLE_DICTIONARY: case PLAIN_DICTIONARY: return true; default: return false; } } } ================================================ FILE: common/src/main/java/org/apache/comet/parquet/CometFileKeyUnwrapper.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.parquet; import java.util.concurrent.ConcurrentHashMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.parquet.crypto.DecryptionKeyRetriever; import org.apache.parquet.crypto.DecryptionPropertiesFactory; import org.apache.parquet.crypto.FileDecryptionProperties; import org.apache.parquet.crypto.ParquetCryptoRuntimeException; // spotless:off /* * Architecture Overview: * * JVM Side | Native Side * ┌─────────────────────────────────────┐ | ┌─────────────────────────────────────┐ * │ CometFileKeyUnwrapper │ | │ Parquet File Reading │ * │ │ | │ │ * │ ┌─────────────────────────────┐ │ | │ ┌─────────────────────────────┐ │ * │ │ hadoopConf │ │ | │ │ file1.parquet │ │ * │ │ (Configuration) │ │ | │ │ file2.parquet │ │ * │ └─────────────────────────────┘ │ | │ │ file3.parquet │ │ * │ │ │ | │ └─────────────────────────────┘ │ * │ ▼ │ | │ │ │ * │ ┌─────────────────────────────┐ │ | │ │ │ * │ │ factoryCache │ │ | │ ▼ │ * │ │ (many-to-one mapping) │ │ | │ ┌─────────────────────────────┐ │ * │ │ │ │ | │ │ Parse file metadata & │ │ * │ │ file1 ──┐ │ │ | │ │ extract keyMetadata │ │ * │ │ file2 ──┼─► DecryptionProps │ │ | │ └─────────────────────────────┘ │ * │ │ file3 ──┘ Factory │ │ | │ │ │ * │ └─────────────────────────────┘ │ | │ │ │ * │ │ │ | │ ▼ │ * │ ▼ │ | │ ╔═════════════════════════════╗ │ * │ ┌─────────────────────────────┐ │ | │ ║ JNI CALL: ║ │ * │ │ retrieverCache │ │ | │ ║ getKey(filePath, ║ │ * │ │ filePath -> KeyRetriever │◄───┼───┼───┼──║ keyMetadata) ║ │ * │ └─────────────────────────────┘ │ | │ ╚═════════════════════════════╝ │ * │ │ │ | │ │ * │ ▼ │ | │ │ * │ ┌─────────────────────────────┐ │ | │ │ * │ │ DecryptionKeyRetriever │ │ | │ │ * │ │ .getKey(keyMetadata) │ │ | │ │ * │ └─────────────────────────────┘ │ | │ │ * │ │ │ | │ │ * │ ▼ │ | │ │ * │ ┌─────────────────────────────┐ │ | │ ┌─────────────────────────────┐ │ * │ │ return key bytes │────┼───┼───┼─►│ Use key for decryption │ │ * │ └─────────────────────────────┘ │ | │ │ of parquet data │ │ * └─────────────────────────────────────┘ | │ └─────────────────────────────┘ │ * | └─────────────────────────────────────┘ * | * JNI Boundary * * Setup Phase (storeDecryptionKeyRetriever): * 1. hadoopConf → DecryptionPropertiesFactory (cached in factoryCache) * 2. Factory + filePath → DecryptionKeyRetriever (cached in retrieverCache) * * Runtime Phase (getKey): * 3. Native code calls getKey(filePath, keyMetadata) ──► JVM * 4. Retrieve cached DecryptionKeyRetriever for filePath * 5. KeyRetriever.getKey(keyMetadata) → decrypted key bytes * 6. Return key bytes ──► Native code for parquet decryption */ // spotless:on /** * Helper class to access DecryptionKeyRetriever.getKey from native code via JNI. This class handles * the complexity of creating and caching properly configured DecryptionKeyRetriever instances using * DecryptionPropertiesFactory. The life of this object is meant to map to a single Comet plan, so * associated with CometExecIterator. */ public class CometFileKeyUnwrapper { // Each file path gets a unique DecryptionKeyRetriever private final ConcurrentHashMap retrieverCache = new ConcurrentHashMap<>(); // Cache the factory since we should be using the same hadoopConf for every file in this scan. private DecryptionPropertiesFactory factory = null; // Cache the hadoopConf just to assert the assumption above. private Configuration conf = null; /** * Normalizes S3 URI schemes to a canonical form. S3 can be accessed via multiple schemes (s3://, * s3a://, s3n://) that refer to the same logical filesystem. This method ensures consistent cache * lookups regardless of which scheme is used. * * @param filePath The file path that may contain an S3 URI * @return The file path with normalized S3 scheme (s3a://) */ private String normalizeS3Scheme(final String filePath) { // Normalize s3:// and s3n:// to s3a:// for consistent cache lookups // This handles the case where ObjectStoreUrl uses s3:// but Spark uses s3a:// String s3Prefix = "s3://"; String s3nPrefix = "s3n://"; if (filePath.startsWith(s3Prefix)) { return "s3a://" + filePath.substring(s3Prefix.length()); } else if (filePath.startsWith(s3nPrefix)) { return "s3a://" + filePath.substring(s3nPrefix.length()); } return filePath; } /** * Creates and stores a DecryptionKeyRetriever instance for the given file path. * * @param filePath The path to the Parquet file * @param hadoopConf The Hadoop Configuration to use for this file path */ public void storeDecryptionKeyRetriever(final String filePath, final Configuration hadoopConf) { final String normalizedPath = normalizeS3Scheme(filePath); // Use DecryptionPropertiesFactory.loadFactory to get the factory and then call // getFileDecryptionProperties if (factory == null) { factory = DecryptionPropertiesFactory.loadFactory(hadoopConf); conf = hadoopConf; } else { // Check the assumption that all files have the same hadoopConf and thus same Factory assert (conf == hadoopConf); } Path path = new Path(filePath); FileDecryptionProperties decryptionProperties = factory.getFileDecryptionProperties(hadoopConf, path); DecryptionKeyRetriever keyRetriever = decryptionProperties.getKeyRetriever(); retrieverCache.put(normalizedPath, keyRetriever); } /** * Gets the decryption key for the given key metadata using the cached DecryptionKeyRetriever for * the specified file path. * * @param filePath The path to the Parquet file * @param keyMetadata The key metadata bytes from the Parquet file * @return The decrypted key bytes * @throws ParquetCryptoRuntimeException if key unwrapping fails */ public byte[] getKey(final String filePath, final byte[] keyMetadata) throws ParquetCryptoRuntimeException { final String normalizedPath = normalizeS3Scheme(filePath); DecryptionKeyRetriever keyRetriever = retrieverCache.get(normalizedPath); if (keyRetriever == null) { throw new ParquetCryptoRuntimeException( "Failed to find DecryptionKeyRetriever for path: " + filePath); } return keyRetriever.getKey(keyMetadata); } } ================================================ FILE: common/src/main/java/org/apache/comet/parquet/CometInputFile.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.parquet; import java.io.IOException; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FutureDataInputStreamBuilder; import org.apache.hadoop.fs.Path; import org.apache.hadoop.util.VersionInfo; import org.apache.parquet.hadoop.util.HadoopStreams; import org.apache.parquet.io.InputFile; import org.apache.parquet.io.SeekableInputStream; /** * A Parquet {@link InputFile} implementation that's similar to {@link * org.apache.parquet.hadoop.util.HadoopInputFile}, but with optimizations introduced in Hadoop 3.x, * for S3 specifically. */ public class CometInputFile implements InputFile { private static final String MAJOR_MINOR_REGEX = "^(\\d+)\\.(\\d+)(\\..*)?$"; private static final Pattern VERSION_MATCHER = Pattern.compile(MAJOR_MINOR_REGEX); private final FileSystem fs; private final FileStatus stat; private final Configuration conf; public static CometInputFile fromPath(Path path, Configuration conf) throws IOException { FileSystem fs = path.getFileSystem(conf); return new CometInputFile(fs, fs.getFileStatus(path), conf); } private CometInputFile(FileSystem fs, FileStatus stat, Configuration conf) { this.fs = fs; this.stat = stat; this.conf = conf; } @Override public long getLength() { return stat.getLen(); } public Configuration getConf() { return this.conf; } public FileSystem getFileSystem() { return this.fs; } public Path getPath() { return stat.getPath(); } @Override public SeekableInputStream newStream() throws IOException { FSDataInputStream stream; try { if (isAtLeastHadoop33()) { // If Hadoop version is >= 3.3.x, we'll use the 'openFile' API which can save a // HEAD request from cloud storages like S3 FutureDataInputStreamBuilder inputStreamBuilder = fs.openFile(stat.getPath()).withFileStatus(stat); if (stat.getPath().toString().startsWith("s3a")) { // Switch to random S3 input policy so that we don't do sequential read on the entire // S3 object. By default, the policy is normal which does sequential read until a back // seek happens, which in our case will never happen. inputStreamBuilder = inputStreamBuilder.opt("fs.s3a.experimental.input.fadvise", "random"); } stream = inputStreamBuilder.build().get(); } else { stream = fs.open(stat.getPath()); } } catch (Exception e) { throw new IOException("Error when opening file " + stat.getPath(), e); } return HadoopStreams.wrap(stream); } public SeekableInputStream newStream(long offset, long length) throws IOException { try { FSDataInputStream stream; if (isAtLeastHadoop33()) { FutureDataInputStreamBuilder inputStreamBuilder = fs.openFile(stat.getPath()).withFileStatus(stat); if (stat.getPath().toString().startsWith("s3a")) { // Switch to random S3 input policy so that we don't do sequential read on the entire // S3 object. By default, the policy is normal which does sequential read until a back // seek happens, which in our case will never happen. // // Also set read ahead length equal to the column chunk length so we don't have to open // multiple S3 http connections. inputStreamBuilder = inputStreamBuilder .opt("fs.s3a.experimental.input.fadvise", "random") .opt("fs.s3a.readahead.range", Long.toString(length)); } stream = inputStreamBuilder.build().get(); } else { stream = fs.open(stat.getPath()); } return HadoopStreams.wrap(stream); } catch (Exception e) { throw new IOException( "Error when opening file " + stat.getPath() + ", offset=" + offset + ", length=" + length, e); } } @Override public String toString() { return stat.getPath().toString(); } private static boolean isAtLeastHadoop33() { String version = VersionInfo.getVersion(); return CometInputFile.isAtLeastHadoop33(version); } static boolean isAtLeastHadoop33(String version) { Matcher matcher = VERSION_MATCHER.matcher(version); if (matcher.matches()) { if (matcher.group(1).equals("3")) { int minorVersion = Integer.parseInt(matcher.group(2)); return minorVersion >= 3; } } return false; } } ================================================ FILE: common/src/main/java/org/apache/comet/parquet/DictionaryPageReader.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.parquet; import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import org.apache.parquet.ParquetReadOptions; import org.apache.parquet.bytes.BytesInput; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.page.DictionaryPage; import org.apache.parquet.column.page.DictionaryPageReadStore; import org.apache.parquet.compression.CompressionCodecFactory; import org.apache.parquet.crypto.AesCipher; import org.apache.parquet.crypto.InternalColumnDecryptionSetup; import org.apache.parquet.crypto.InternalFileDecryptor; import org.apache.parquet.crypto.ModuleCipherFactory; import org.apache.parquet.format.BlockCipher; import org.apache.parquet.format.DictionaryPageHeader; import org.apache.parquet.format.PageHeader; import org.apache.parquet.format.Util; import org.apache.parquet.hadoop.metadata.BlockMetaData; import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; import org.apache.parquet.io.ParquetDecodingException; import org.apache.parquet.io.SeekableInputStream; public class DictionaryPageReader implements DictionaryPageReadStore { private final Map> cache; private final InternalFileDecryptor fileDecryptor; private final SeekableInputStream inputStream; private final ParquetReadOptions options; private final Map columns; DictionaryPageReader( BlockMetaData block, InternalFileDecryptor fileDecryptor, SeekableInputStream inputStream, ParquetReadOptions options) { this.columns = new HashMap<>(); this.cache = new ConcurrentHashMap<>(); this.fileDecryptor = fileDecryptor; this.inputStream = inputStream; this.options = options; for (ColumnChunkMetaData column : block.getColumns()) { columns.put(column.getPath().toDotString(), column); } } @Override public DictionaryPage readDictionaryPage(ColumnDescriptor descriptor) { String dotPath = String.join(".", descriptor.getPath()); ColumnChunkMetaData column = columns.get(dotPath); if (column == null) { throw new ParquetDecodingException("Failed to load dictionary, unknown column: " + dotPath); } return cache .computeIfAbsent( dotPath, key -> { try { final DictionaryPage dict = column.hasDictionaryPage() ? readDictionary(column) : null; // Copy the dictionary to ensure it can be reused if it is returned // more than once. This can happen when a DictionaryFilter has two or // more predicates for the same column. Cache misses as well. return (dict != null) ? Optional.of(reusableCopy(dict)) : Optional.empty(); } catch (IOException e) { throw new ParquetDecodingException("Failed to read dictionary", e); } }) .orElse(null); } DictionaryPage readDictionary(ColumnChunkMetaData meta) throws IOException { if (!meta.hasDictionaryPage()) { return null; } if (inputStream.getPos() != meta.getStartingPos()) { inputStream.seek(meta.getStartingPos()); } boolean encryptedColumn = false; InternalColumnDecryptionSetup columnDecryptionSetup = null; byte[] dictionaryPageAAD = null; BlockCipher.Decryptor pageDecryptor = null; if (null != fileDecryptor && !fileDecryptor.plaintextFile()) { columnDecryptionSetup = fileDecryptor.getColumnSetup(meta.getPath()); if (columnDecryptionSetup.isEncrypted()) { encryptedColumn = true; } } PageHeader pageHeader; if (!encryptedColumn) { pageHeader = Util.readPageHeader(inputStream); } else { byte[] dictionaryPageHeaderAAD = AesCipher.createModuleAAD( fileDecryptor.getFileAAD(), ModuleCipherFactory.ModuleType.DictionaryPageHeader, meta.getRowGroupOrdinal(), columnDecryptionSetup.getOrdinal(), -1); pageHeader = Util.readPageHeader( inputStream, columnDecryptionSetup.getMetaDataDecryptor(), dictionaryPageHeaderAAD); dictionaryPageAAD = AesCipher.createModuleAAD( fileDecryptor.getFileAAD(), ModuleCipherFactory.ModuleType.DictionaryPage, meta.getRowGroupOrdinal(), columnDecryptionSetup.getOrdinal(), -1); pageDecryptor = columnDecryptionSetup.getDataDecryptor(); } if (!pageHeader.isSetDictionary_page_header()) { return null; } DictionaryPage compressedPage = readCompressedDictionary(pageHeader, inputStream, pageDecryptor, dictionaryPageAAD); CompressionCodecFactory.BytesInputDecompressor decompressor = options.getCodecFactory().getDecompressor(meta.getCodec()); return new DictionaryPage( decompressor.decompress(compressedPage.getBytes(), compressedPage.getUncompressedSize()), compressedPage.getDictionarySize(), compressedPage.getEncoding()); } private DictionaryPage readCompressedDictionary( PageHeader pageHeader, SeekableInputStream fin, BlockCipher.Decryptor pageDecryptor, byte[] dictionaryPageAAD) throws IOException { DictionaryPageHeader dictHeader = pageHeader.getDictionary_page_header(); int uncompressedPageSize = pageHeader.getUncompressed_page_size(); int compressedPageSize = pageHeader.getCompressed_page_size(); byte[] dictPageBytes = new byte[compressedPageSize]; fin.readFully(dictPageBytes); BytesInput bin = BytesInput.from(dictPageBytes); if (null != pageDecryptor) { bin = BytesInput.from(pageDecryptor.decrypt(bin.toByteArray(), dictionaryPageAAD)); } return new DictionaryPage( bin, uncompressedPageSize, dictHeader.getNum_values(), org.apache.parquet.column.Encoding.valueOf(dictHeader.getEncoding().name())); } private static DictionaryPage reusableCopy(DictionaryPage dict) throws IOException { return new DictionaryPage( BytesInput.from(dict.getBytes().toByteArray()), dict.getDictionarySize(), dict.getEncoding()); } } ================================================ FILE: common/src/main/java/org/apache/comet/parquet/FileReader.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.parquet; import java.io.Closeable; import java.io.IOException; import java.io.InputStream; import java.lang.reflect.Method; import java.net.URI; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.zip.CRC32; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.parquet.HadoopReadOptions; import org.apache.parquet.ParquetReadOptions; import org.apache.parquet.Preconditions; import org.apache.parquet.bytes.ByteBufferInputStream; import org.apache.parquet.bytes.BytesInput; import org.apache.parquet.bytes.BytesUtils; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.page.DataPage; import org.apache.parquet.column.page.DataPageV1; import org.apache.parquet.column.page.DataPageV2; import org.apache.parquet.column.page.DictionaryPage; import org.apache.parquet.column.page.PageReadStore; import org.apache.parquet.compression.CompressionCodecFactory; import org.apache.parquet.crypto.AesCipher; import org.apache.parquet.crypto.EncryptionPropertiesFactory; import org.apache.parquet.crypto.FileDecryptionProperties; import org.apache.parquet.crypto.InternalColumnDecryptionSetup; import org.apache.parquet.crypto.InternalFileDecryptor; import org.apache.parquet.crypto.ModuleCipherFactory; import org.apache.parquet.crypto.ParquetCryptoRuntimeException; import org.apache.parquet.filter2.compat.FilterCompat; import org.apache.parquet.format.BlockCipher; import org.apache.parquet.format.DataPageHeader; import org.apache.parquet.format.DataPageHeaderV2; import org.apache.parquet.format.DictionaryPageHeader; import org.apache.parquet.format.FileCryptoMetaData; import org.apache.parquet.format.PageHeader; import org.apache.parquet.format.Util; import org.apache.parquet.format.converter.ParquetMetadataConverter; import org.apache.parquet.hadoop.ParquetInputFormat; import org.apache.parquet.hadoop.metadata.BlockMetaData; import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; import org.apache.parquet.hadoop.metadata.ColumnPath; import org.apache.parquet.hadoop.metadata.FileMetaData; import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.apache.parquet.hadoop.util.counters.BenchmarkCounter; import org.apache.parquet.internal.column.columnindex.OffsetIndex; import org.apache.parquet.internal.filter2.columnindex.ColumnIndexFilter; import org.apache.parquet.internal.filter2.columnindex.ColumnIndexStore; import org.apache.parquet.internal.filter2.columnindex.RowRanges; import org.apache.parquet.io.InputFile; import org.apache.parquet.io.ParquetDecodingException; import org.apache.parquet.io.SeekableInputStream; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.PrimitiveType; import org.apache.spark.sql.execution.metric.SQLMetric; import org.apache.comet.IcebergApi; import static org.apache.parquet.hadoop.ParquetFileWriter.EFMAGIC; import static org.apache.parquet.hadoop.ParquetFileWriter.MAGIC; import static org.apache.comet.parquet.RowGroupFilter.FilterLevel.BLOOMFILTER; import static org.apache.comet.parquet.RowGroupFilter.FilterLevel.DICTIONARY; import static org.apache.comet.parquet.RowGroupFilter.FilterLevel.STATISTICS; /** * A Parquet file reader. Mostly followed {@code ParquetFileReader} in {@code parquet-mr}, but with * customizations & optimizations for Comet. */ @IcebergApi public class FileReader implements Closeable { private static final Logger LOG = LoggerFactory.getLogger(FileReader.class); private final ParquetMetadataConverter converter; private final SeekableInputStream f; private final InputFile file; private final Map metrics; private final Map paths = new HashMap<>(); private final FileMetaData fileMetaData; // may be null private final List blocks; private final List blockIndexStores; private final List blockRowRanges; private final CRC32 crc; private final ParquetMetadata footer; /** * Read configurations come from two options: - options: these are options defined & specified * from 'parquet-mr' library - cometOptions: these are Comet-specific options, for the features * introduced in Comet's Parquet implementation */ private final ParquetReadOptions options; private final ReadOptions cometOptions; private int currentBlock = 0; private RowGroupReader currentRowGroup = null; private InternalFileDecryptor fileDecryptor; FileReader(InputFile file, ParquetReadOptions options, ReadOptions cometOptions) throws IOException { this(file, null, options, cometOptions, null); } /** This constructor is called from Apache Iceberg. */ @IcebergApi public FileReader( WrappedInputFile file, ReadOptions cometOptions, Map properties, Long start, Long length, byte[] fileEncryptionKey, byte[] fileAADPrefix) throws IOException { ParquetReadOptions options = buildParquetReadOptions( new Configuration(), properties, start, length, fileEncryptionKey, fileAADPrefix); this.converter = new ParquetMetadataConverter(options); this.file = file; this.f = file.newStream(); this.options = options; this.cometOptions = cometOptions; this.metrics = null; try { this.footer = readFooter(file, options, f, converter); } catch (Exception e) { // In case that reading footer throws an exception in the constructor, the new stream // should be closed. Otherwise, there's no way to close this outside. f.close(); throw e; } this.fileMetaData = footer.getFileMetaData(); this.fileDecryptor = fileMetaData.getFileDecryptor(); // must be called before filterRowGroups! if (null != fileDecryptor && fileDecryptor.plaintextFile()) { this.fileDecryptor = null; // Plaintext file. No need in decryptor } this.blocks = footer.getBlocks(); // filter row group in iceberg this.blockIndexStores = listWithNulls(this.blocks.size()); this.blockRowRanges = listWithNulls(this.blocks.size()); for (ColumnDescriptor col : footer.getFileMetaData().getSchema().getColumns()) { paths.put(ColumnPath.get(col.getPath()), col); } this.crc = options.usePageChecksumVerification() ? new CRC32() : null; } FileReader( InputFile file, ParquetReadOptions options, ReadOptions cometOptions, Map metrics) throws IOException { this(file, null, options, cometOptions, metrics); } FileReader( InputFile file, ParquetMetadata footer, ParquetReadOptions options, ReadOptions cometOptions, Map metrics) throws IOException { this.converter = new ParquetMetadataConverter(options); this.file = file; this.f = file.newStream(); this.options = options; this.cometOptions = cometOptions; this.metrics = metrics; if (footer == null) { try { footer = readFooter(file, options, f, converter); } catch (Exception e) { // In case that reading footer throws an exception in the constructor, the new stream // should be closed. Otherwise, there's no way to close this outside. f.close(); throw e; } } this.footer = footer; this.fileMetaData = footer.getFileMetaData(); this.fileDecryptor = fileMetaData.getFileDecryptor(); // must be called before filterRowGroups! if (null != fileDecryptor && fileDecryptor.plaintextFile()) { this.fileDecryptor = null; // Plaintext file. No need in decryptor } this.blocks = filterRowGroups(footer.getBlocks()); this.blockIndexStores = listWithNulls(this.blocks.size()); this.blockRowRanges = listWithNulls(this.blocks.size()); for (ColumnDescriptor col : footer.getFileMetaData().getSchema().getColumns()) { paths.put(ColumnPath.get(col.getPath()), col); } this.crc = options.usePageChecksumVerification() ? new CRC32() : null; } /** Returns the footer of the Parquet file being read. */ ParquetMetadata getFooter() { return this.footer; } /** Returns the metadata of the Parquet file being read. */ FileMetaData getFileMetaData() { return this.fileMetaData; } /** Returns the input stream of the Parquet file being read. */ public SeekableInputStream getInputStream() { return this.f; } /** Returns the Parquet options for reading the file. */ public ParquetReadOptions getOptions() { return this.options; } /** Returns all the row groups of this reader (after applying row group filtering). */ public List getRowGroups() { return blocks; } /** Sets the projected columns to be read later via {@link #readNextRowGroup()} */ public void setRequestedSchema(List projection) { paths.clear(); for (ColumnDescriptor col : projection) { paths.put(ColumnPath.get(col.getPath()), col); } } /** This method is called from Apache Iceberg. */ @IcebergApi public void setRequestedSchemaFromSpecs(List specList) { paths.clear(); for (ParquetColumnSpec colSpec : specList) { ColumnDescriptor descriptor = Utils.buildColumnDescriptor(colSpec); paths.put(ColumnPath.get(colSpec.getPath()), descriptor); } } private static ParquetReadOptions buildParquetReadOptions( Configuration conf, Map properties, Long start, Long length, byte[] fileEncryptionKey, byte[] fileAADPrefix) { // Iceberg remove these read properties when building the ParquetReadOptions. // We want build the exact same ParquetReadOptions as Iceberg's. Collection readPropertiesToRemove = Set.of( ParquetInputFormat.UNBOUND_RECORD_FILTER, ParquetInputFormat.FILTER_PREDICATE, ParquetInputFormat.READ_SUPPORT_CLASS, EncryptionPropertiesFactory.CRYPTO_FACTORY_CLASS_PROPERTY_NAME); for (String property : readPropertiesToRemove) { conf.unset(property); } ParquetReadOptions.Builder optionsBuilder = HadoopReadOptions.builder(conf); for (Map.Entry entry : properties.entrySet()) { optionsBuilder.set(entry.getKey(), entry.getValue()); } if (start != null && length != null) { optionsBuilder.withRange(start, start + length); } if (fileEncryptionKey != null) { FileDecryptionProperties fileDecryptionProperties = FileDecryptionProperties.builder() .withFooterKey(fileEncryptionKey) .withAADPrefix(fileAADPrefix) .build(); optionsBuilder.withDecryption(fileDecryptionProperties); } return optionsBuilder.build(); } /** * Gets the total number of records across all row groups (after applying row group filtering). */ public long getRecordCount() { long total = 0; for (BlockMetaData block : blocks) { total += block.getRowCount(); } return total; } /** * Gets the total number of records across all row groups (after applying both row group filtering * and page-level column index filtering). */ public long getFilteredRecordCount() { if (!options.useColumnIndexFilter() || !FilterCompat.isFilteringRequired(options.getRecordFilter())) { return getRecordCount(); } long total = 0; for (int i = 0, n = blocks.size(); i < n; ++i) { total += getRowRanges(i).rowCount(); } return total; } /** Skips the next row group. Returns false if there's no row group to skip. Otherwise, true. */ @IcebergApi public boolean skipNextRowGroup() { return advanceToNextBlock(); } /** * Returns the next row group to read (after applying row group filtering), or null if there's no * more row group. */ @IcebergApi public RowGroupReader readNextRowGroup() throws IOException { if (currentBlock == blocks.size()) { return null; } BlockMetaData block = blocks.get(currentBlock); if (block.getRowCount() == 0) { throw new RuntimeException("Illegal row group of 0 rows"); } this.currentRowGroup = new RowGroupReader(block.getRowCount(), block.getRowIndexOffset()); // prepare the list of consecutive parts to read them in one scan List allParts = new ArrayList<>(); ConsecutivePartList currentParts = null; for (ColumnChunkMetaData mc : block.getColumns()) { ColumnPath pathKey = mc.getPath(); ColumnDescriptor columnDescriptor = paths.get(pathKey); if (columnDescriptor != null) { BenchmarkCounter.incrementTotalBytes(mc.getTotalSize()); long startingPos = mc.getStartingPos(); boolean mergeRanges = cometOptions.isIOMergeRangesEnabled(); int mergeRangeDelta = cometOptions.getIOMergeRangesDelta(); // start a new list if - // it is the first part or // the part is consecutive or // the part is not consecutive but within the merge range if (currentParts == null || (!mergeRanges && currentParts.endPos() != startingPos) || (mergeRanges && startingPos - currentParts.endPos() > mergeRangeDelta)) { currentParts = new ConsecutivePartList(startingPos); allParts.add(currentParts); } // if we are in a consecutive part list and there is a gap in between the parts, // we treat the gap as a skippable chunk long delta = startingPos - currentParts.endPos(); if (mergeRanges && delta > 0 && delta <= mergeRangeDelta) { // add a chunk that will be skipped because it has no column descriptor currentParts.addChunk(new ChunkDescriptor(null, null, startingPos, delta)); } currentParts.addChunk( new ChunkDescriptor(columnDescriptor, mc, startingPos, mc.getTotalSize())); } } // actually read all the chunks return readChunks(block, allParts, new ChunkListBuilder()); } /** * Returns the next row group to read (after applying both row group filtering and page level * column index filtering), or null if there's no more row group. */ public PageReadStore readNextFilteredRowGroup() throws IOException { if (currentBlock == blocks.size()) { return null; } if (!options.useColumnIndexFilter() || !FilterCompat.isFilteringRequired(options.getRecordFilter())) { return readNextRowGroup(); } BlockMetaData block = blocks.get(currentBlock); if (block.getRowCount() == 0) { throw new RuntimeException("Illegal row group of 0 rows"); } ColumnIndexStore ciStore = getColumnIndexReader(currentBlock); RowRanges rowRanges = getRowRanges(currentBlock); long rowCount = rowRanges.rowCount(); if (rowCount == 0) { // There are no matching rows -> skipping this row-group advanceToNextBlock(); return readNextFilteredRowGroup(); } if (rowCount == block.getRowCount()) { // All rows are matching -> fall back to the non-filtering path return readNextRowGroup(); } this.currentRowGroup = new RowGroupReader(rowRanges); // prepare the list of consecutive parts to read them in one scan ChunkListBuilder builder = new ChunkListBuilder(); List allParts = new ArrayList<>(); ConsecutivePartList currentParts = null; for (ColumnChunkMetaData mc : block.getColumns()) { ColumnPath pathKey = mc.getPath(); ColumnDescriptor columnDescriptor = paths.get(pathKey); if (columnDescriptor != null) { OffsetIndex offsetIndex = ciStore.getOffsetIndex(mc.getPath()); IndexFilter indexFilter = new IndexFilter(rowRanges, offsetIndex, block.getRowCount()); OffsetIndex filteredOffsetIndex = indexFilter.filterOffsetIndex(); for (IndexFilter.OffsetRange range : indexFilter.calculateOffsetRanges(filteredOffsetIndex, mc)) { BenchmarkCounter.incrementTotalBytes(range.length); long startingPos = range.offset; // first part or not consecutive => new list if (currentParts == null || currentParts.endPos() != startingPos) { currentParts = new ConsecutivePartList(startingPos); allParts.add(currentParts); } ChunkDescriptor chunkDescriptor = new ChunkDescriptor(columnDescriptor, mc, startingPos, range.length); currentParts.addChunk(chunkDescriptor); builder.setOffsetIndex(chunkDescriptor, filteredOffsetIndex); } } } // actually read all the chunks return readChunks(block, allParts, builder); } // Visible for testing ColumnIndexReader getColumnIndexReader(int blockIndex) { ColumnIndexReader ciStore = blockIndexStores.get(blockIndex); if (ciStore == null) { ciStore = ColumnIndexReader.create(blocks.get(blockIndex), paths.keySet(), fileDecryptor, f); blockIndexStores.set(blockIndex, ciStore); } return ciStore; } private RowGroupReader readChunks( BlockMetaData block, List allParts, ChunkListBuilder builder) throws IOException { if (shouldReadParallel()) { readAllPartsParallel(allParts, builder); } else { for (ConsecutivePartList consecutiveChunks : allParts) { consecutiveChunks.readAll(f, builder); } } for (Chunk chunk : builder.build()) { readChunkPages(chunk, block); } advanceToNextBlock(); return currentRowGroup; } private boolean shouldReadParallel() { if (file instanceof CometInputFile) { URI uri = ((CometInputFile) file).getPath().toUri(); return shouldReadParallel(cometOptions, uri.getScheme()); } return false; } static boolean shouldReadParallel(ReadOptions options, String scheme) { return options.isParallelIOEnabled() && shouldReadParallelForScheme(scheme); } private static boolean shouldReadParallelForScheme(String scheme) { if (scheme == null) { return false; } switch (scheme) { case "s3a": // Only enable parallel read for S3, so far. return true; default: return false; } } static class ReadRange { long offset = 0; long length = 0; List buffers = new ArrayList<>(); @Override public String toString() { return "ReadRange{" + "offset=" + offset + ", length=" + length + ", numBuffers=" + buffers.size() + '}'; } } List getReadRanges(List allParts, int nBuffers) { int nThreads = cometOptions.parallelIOThreadPoolSize(); long buffersPerThread = nBuffers / nThreads + 1; boolean adjustSkew = cometOptions.adjustReadRangesSkew(); List allRanges = new ArrayList<>(); for (ConsecutivePartList consecutiveChunk : allParts) { ReadRange readRange = null; long offset = consecutiveChunk.offset; for (int i = 0; i < consecutiveChunk.buffers.size(); i++) { if ((adjustSkew && (i % buffersPerThread == 0)) || i == 0) { readRange = new ReadRange(); allRanges.add(readRange); readRange.offset = offset; } ByteBuffer b = consecutiveChunk.buffers.get(i); readRange.length += b.capacity(); readRange.buffers.add(b); offset += b.capacity(); } } if (LOG.isDebugEnabled()) { StringBuilder sb = new StringBuilder(); for (int i = 0; i < allRanges.size(); i++) { sb.append(allRanges.get(i).toString()); if (i < allRanges.size() - 1) { sb.append(","); } } LOG.debug("Read Ranges: {}", sb); } return allRanges; } private void readAllRangesParallel(List allRanges) { int nThreads = cometOptions.parallelIOThreadPoolSize(); ExecutorService threadPool = CometFileReaderThreadPool.getOrCreateThreadPool(nThreads); List> futures = new ArrayList<>(); for (ReadRange readRange : allRanges) { futures.add( threadPool.submit( () -> { SeekableInputStream inputStream = null; try { if (file instanceof CometInputFile) { // limit the max read ahead to length of the range inputStream = (((CometInputFile) file).newStream(readRange.offset, readRange.length)); LOG.debug( "Opened new input file: {}, at offset: {}", ((CometInputFile) file).getPath().getName(), readRange.offset); } else { inputStream = file.newStream(); } long curPos = readRange.offset; for (ByteBuffer buffer : readRange.buffers) { inputStream.seek(curPos); LOG.debug( "Thread: {} Offset: {} Size: {}", Thread.currentThread().getId(), curPos, buffer.capacity()); inputStream.readFully(buffer); buffer.flip(); curPos += buffer.capacity(); } // for } finally { if (inputStream != null) { inputStream.close(); } } return null; })); } for (Future future : futures) { try { future.get(); } catch (InterruptedException | ExecutionException e) { throw new RuntimeException(e); } } } /** * Read all the consecutive part list objects in parallel. * * @param allParts all consecutive parts * @param builder chunk list builder */ public void readAllPartsParallel(List allParts, ChunkListBuilder builder) throws IOException { int nBuffers = 0; for (ConsecutivePartList consecutiveChunks : allParts) { consecutiveChunks.allocateReadBuffers(); nBuffers += consecutiveChunks.buffers.size(); } List allRanges = getReadRanges(allParts, nBuffers); long startNs = System.nanoTime(); readAllRangesParallel(allRanges); for (ConsecutivePartList consecutiveChunks : allParts) { consecutiveChunks.setReadMetrics(startNs); ByteBufferInputStream stream; stream = ByteBufferInputStream.wrap(consecutiveChunks.buffers); // report in a counter the data we just scanned BenchmarkCounter.incrementBytesRead(consecutiveChunks.length); for (int i = 0; i < consecutiveChunks.chunks.size(); i++) { ChunkDescriptor descriptor = consecutiveChunks.chunks.get(i); if (descriptor.col != null) { builder.add(descriptor, stream.sliceBuffers(descriptor.size)); } else { stream.skipFully(descriptor.size); } } } } private void readChunkPages(Chunk chunk, BlockMetaData block) throws IOException { if (fileDecryptor == null || fileDecryptor.plaintextFile()) { currentRowGroup.addColumn(chunk.descriptor.col, chunk.readAllPages()); return; } // Encrypted file ColumnPath columnPath = ColumnPath.get(chunk.descriptor.col.getPath()); InternalColumnDecryptionSetup columnDecryptionSetup = fileDecryptor.getColumnSetup(columnPath); if (!columnDecryptionSetup.isEncrypted()) { // plaintext column currentRowGroup.addColumn(chunk.descriptor.col, chunk.readAllPages()); } else { // encrypted column currentRowGroup.addColumn( chunk.descriptor.col, chunk.readAllPages( columnDecryptionSetup.getMetaDataDecryptor(), columnDecryptionSetup.getDataDecryptor(), fileDecryptor.getFileAAD(), block.getOrdinal(), columnDecryptionSetup.getOrdinal())); } } private boolean advanceToNextBlock() { if (currentBlock == blocks.size()) { return false; } // update the current block and instantiate a dictionary reader for it ++currentBlock; return true; } public long[] getRowIndices() { return getRowIndices(blocks); } public static long[] getRowIndices(List blocks) { long[] rowIndices = new long[blocks.size() * 2]; for (int i = 0, n = blocks.size(); i < n; i++) { BlockMetaData block = blocks.get(i); rowIndices[i * 2] = getRowIndexOffset(block); rowIndices[i * 2 + 1] = block.getRowCount(); } return rowIndices; } // Uses reflection to get row index offset from a Parquet block metadata. // // The reason reflection is used here is that some Spark versions still depend on a // Parquet version where the method `getRowIndexOffset` is not public. public static long getRowIndexOffset(BlockMetaData metaData) { try { Method method = BlockMetaData.class.getMethod("getRowIndexOffset"); method.setAccessible(true); return (long) method.invoke(metaData); } catch (Exception e) { throw new RuntimeException("Error when calling getRowIndexOffset", e); } } private RowRanges getRowRanges(int blockIndex) { Preconditions.checkState( FilterCompat.isFilteringRequired(options.getRecordFilter()), "Should not be invoked if filter is null or NOOP"); RowRanges rowRanges = blockRowRanges.get(blockIndex); if (rowRanges == null) { rowRanges = ColumnIndexFilter.calculateRowRanges( options.getRecordFilter(), getColumnIndexReader(blockIndex), paths.keySet(), blocks.get(blockIndex).getRowCount()); blockRowRanges.set(blockIndex, rowRanges); } return rowRanges; } private static ParquetMetadata readFooter( InputFile file, ParquetReadOptions options, SeekableInputStream f, ParquetMetadataConverter converter) throws IOException { long fileLen = file.getLength(); String filePath = file.toString(); LOG.debug("File length {}", fileLen); int FOOTER_LENGTH_SIZE = 4; // MAGIC + data + footer + footerIndex + MAGIC if (fileLen < MAGIC.length + FOOTER_LENGTH_SIZE + MAGIC.length) { throw new RuntimeException( filePath + " is not a Parquet file (length is too low: " + fileLen + ")"); } // Read footer length and magic string - with a single seek byte[] magic = new byte[MAGIC.length]; long fileMetadataLengthIndex = fileLen - magic.length - FOOTER_LENGTH_SIZE; LOG.debug("reading footer index at {}", fileMetadataLengthIndex); f.seek(fileMetadataLengthIndex); int fileMetadataLength = BytesUtils.readIntLittleEndian(f); f.readFully(magic); boolean encryptedFooterMode; if (Arrays.equals(MAGIC, magic)) { encryptedFooterMode = false; } else if (Arrays.equals(EFMAGIC, magic)) { encryptedFooterMode = true; } else { throw new RuntimeException( filePath + " is not a Parquet file. Expected magic number " + "at tail, but found " + Arrays.toString(magic)); } long fileMetadataIndex = fileMetadataLengthIndex - fileMetadataLength; LOG.debug("read footer length: {}, footer index: {}", fileMetadataLength, fileMetadataIndex); if (fileMetadataIndex < magic.length || fileMetadataIndex >= fileMetadataLengthIndex) { throw new RuntimeException( "corrupted file: the footer index is not within the file: " + fileMetadataIndex); } f.seek(fileMetadataIndex); FileDecryptionProperties fileDecryptionProperties = options.getDecryptionProperties(); InternalFileDecryptor fileDecryptor = null; if (null != fileDecryptionProperties) { fileDecryptor = new InternalFileDecryptor(fileDecryptionProperties); } // Read all the footer bytes in one time to avoid multiple read operations, // since it can be pretty time consuming for a single read operation in HDFS. byte[] footerBytes = new byte[fileMetadataLength]; f.readFully(footerBytes); ByteBuffer footerBytesBuffer = ByteBuffer.wrap(footerBytes); LOG.debug("Finished to read all footer bytes."); InputStream footerBytesStream = ByteBufferInputStream.wrap(footerBytesBuffer); // Regular file, or encrypted file with plaintext footer if (!encryptedFooterMode) { return converter.readParquetMetadata( footerBytesStream, options.getMetadataFilter(), fileDecryptor, false, fileMetadataLength); } // Encrypted file with encrypted footer if (fileDecryptor == null) { throw new ParquetCryptoRuntimeException( "Trying to read file with encrypted footer. " + "No keys available"); } FileCryptoMetaData fileCryptoMetaData = Util.readFileCryptoMetaData(footerBytesStream); fileDecryptor.setFileCryptoMetaData( fileCryptoMetaData.getEncryption_algorithm(), true, fileCryptoMetaData.getKey_metadata()); // footer length is required only for signed plaintext footers return converter.readParquetMetadata( footerBytesStream, options.getMetadataFilter(), fileDecryptor, true, 0); } private List filterRowGroups(List blocks) { return filterRowGroups(options, blocks, this); } public static List filterRowGroups( ParquetReadOptions options, List blocks, FileReader fileReader) { FilterCompat.Filter recordFilter = options.getRecordFilter(); if (FilterCompat.isFilteringRequired(recordFilter)) { // set up data filters based on configured levels List levels = new ArrayList<>(); if (options.useStatsFilter()) { levels.add(STATISTICS); } if (options.useDictionaryFilter()) { levels.add(DICTIONARY); } if (options.useBloomFilter()) { levels.add(BLOOMFILTER); } return RowGroupFilter.filterRowGroups(levels, recordFilter, blocks, fileReader); } return blocks; } public static List filterRowGroups( ParquetReadOptions options, List blocks, MessageType schema) { FilterCompat.Filter recordFilter = options.getRecordFilter(); if (FilterCompat.isFilteringRequired(recordFilter)) { // set up data filters based on configured levels List levels = new ArrayList<>(); if (options.useStatsFilter()) { levels.add(STATISTICS); } if (options.useDictionaryFilter()) { levels.add(DICTIONARY); } if (options.useBloomFilter()) { levels.add(BLOOMFILTER); } return RowGroupFilter.filterRowGroups(levels, recordFilter, blocks, schema); } return blocks; } private static List listWithNulls(int size) { return Stream.generate(() -> (T) null).limit(size).collect(Collectors.toList()); } public void closeStream() throws IOException { if (f != null) { f.close(); } } @IcebergApi @Override public void close() throws IOException { try { if (f != null) { f.close(); } } finally { options.getCodecFactory().release(); } } /** * Builder to concatenate the buffers of the discontinuous parts for the same column. These parts * are generated as a result of the column-index based filtering when some pages might be skipped * at reading. */ private class ChunkListBuilder { private class ChunkData { final List buffers = new ArrayList<>(); OffsetIndex offsetIndex; } private final Map map = new HashMap<>(); void add(ChunkDescriptor descriptor, List buffers) { ChunkListBuilder.ChunkData data = map.get(descriptor); if (data == null) { data = new ChunkData(); map.put(descriptor, data); } data.buffers.addAll(buffers); } void setOffsetIndex(ChunkDescriptor descriptor, OffsetIndex offsetIndex) { ChunkData data = map.get(descriptor); if (data == null) { data = new ChunkData(); map.put(descriptor, data); } data.offsetIndex = offsetIndex; } List build() { List chunks = new ArrayList<>(); for (Map.Entry entry : map.entrySet()) { ChunkDescriptor descriptor = entry.getKey(); ChunkData data = entry.getValue(); chunks.add(new Chunk(descriptor, data.buffers, data.offsetIndex)); } return chunks; } } /** The data for a column chunk */ private class Chunk { private final ChunkDescriptor descriptor; private final ByteBufferInputStream stream; final OffsetIndex offsetIndex; /** * @param descriptor descriptor for the chunk * @param buffers ByteBuffers that contain the chunk * @param offsetIndex the offset index for this column; might be null */ Chunk(ChunkDescriptor descriptor, List buffers, OffsetIndex offsetIndex) { this.descriptor = descriptor; this.stream = ByteBufferInputStream.wrap(buffers); this.offsetIndex = offsetIndex; } protected PageHeader readPageHeader(BlockCipher.Decryptor blockDecryptor, byte[] pageHeaderAAD) throws IOException { return Util.readPageHeader(stream, blockDecryptor, pageHeaderAAD); } /** * Calculate checksum of input bytes, throw decoding exception if it does not match the provided * reference crc */ private void verifyCrc(int referenceCrc, byte[] bytes, String exceptionMsg) { crc.reset(); crc.update(bytes); if (crc.getValue() != ((long) referenceCrc & 0xffffffffL)) { throw new ParquetDecodingException(exceptionMsg); } } private ColumnPageReader readAllPages() throws IOException { return readAllPages(null, null, null, -1, -1); } private ColumnPageReader readAllPages( BlockCipher.Decryptor headerBlockDecryptor, BlockCipher.Decryptor pageBlockDecryptor, byte[] aadPrefix, int rowGroupOrdinal, int columnOrdinal) throws IOException { List pagesInChunk = new ArrayList<>(); DictionaryPage dictionaryPage = null; PrimitiveType type = fileMetaData.getSchema().getType(descriptor.col.getPath()).asPrimitiveType(); long valuesCountReadSoFar = 0; int dataPageCountReadSoFar = 0; byte[] dataPageHeaderAAD = null; if (null != headerBlockDecryptor) { dataPageHeaderAAD = AesCipher.createModuleAAD( aadPrefix, ModuleCipherFactory.ModuleType.DataPageHeader, rowGroupOrdinal, columnOrdinal, getPageOrdinal(dataPageCountReadSoFar)); } while (hasMorePages(valuesCountReadSoFar, dataPageCountReadSoFar)) { byte[] pageHeaderAAD = dataPageHeaderAAD; if (null != headerBlockDecryptor) { // Important: this verifies file integrity (makes sure dictionary page had not been // removed) if (null == dictionaryPage && descriptor.metadata.hasDictionaryPage()) { pageHeaderAAD = AesCipher.createModuleAAD( aadPrefix, ModuleCipherFactory.ModuleType.DictionaryPageHeader, rowGroupOrdinal, columnOrdinal, -1); } else { int pageOrdinal = getPageOrdinal(dataPageCountReadSoFar); AesCipher.quickUpdatePageAAD(dataPageHeaderAAD, pageOrdinal); } } PageHeader pageHeader = readPageHeader(headerBlockDecryptor, pageHeaderAAD); int uncompressedPageSize = pageHeader.getUncompressed_page_size(); int compressedPageSize = pageHeader.getCompressed_page_size(); final BytesInput pageBytes; switch (pageHeader.type) { case DICTIONARY_PAGE: // there is only one dictionary page per column chunk if (dictionaryPage != null) { throw new ParquetDecodingException( "more than one dictionary page in column " + descriptor.col); } pageBytes = this.readAsBytesInput(compressedPageSize); if (options.usePageChecksumVerification() && pageHeader.isSetCrc()) { verifyCrc( pageHeader.getCrc(), pageBytes.toByteArray(), "could not verify dictionary page integrity, CRC checksum verification failed"); } DictionaryPageHeader dicHeader = pageHeader.getDictionary_page_header(); dictionaryPage = new DictionaryPage( pageBytes, uncompressedPageSize, dicHeader.getNum_values(), converter.getEncoding(dicHeader.getEncoding())); // Copy crc to new page, used for testing if (pageHeader.isSetCrc()) { dictionaryPage.setCrc(pageHeader.getCrc()); } break; case DATA_PAGE: DataPageHeader dataHeaderV1 = pageHeader.getData_page_header(); pageBytes = this.readAsBytesInput(compressedPageSize); if (options.usePageChecksumVerification() && pageHeader.isSetCrc()) { verifyCrc( pageHeader.getCrc(), pageBytes.toByteArray(), "could not verify page integrity, CRC checksum verification failed"); } DataPageV1 dataPageV1 = new DataPageV1( pageBytes, dataHeaderV1.getNum_values(), uncompressedPageSize, converter.fromParquetStatistics( getFileMetaData().getCreatedBy(), dataHeaderV1.getStatistics(), type), converter.getEncoding(dataHeaderV1.getRepetition_level_encoding()), converter.getEncoding(dataHeaderV1.getDefinition_level_encoding()), converter.getEncoding(dataHeaderV1.getEncoding())); // Copy crc to new page, used for testing if (pageHeader.isSetCrc()) { dataPageV1.setCrc(pageHeader.getCrc()); } pagesInChunk.add(dataPageV1); valuesCountReadSoFar += dataHeaderV1.getNum_values(); ++dataPageCountReadSoFar; break; case DATA_PAGE_V2: DataPageHeaderV2 dataHeaderV2 = pageHeader.getData_page_header_v2(); int dataSize = compressedPageSize - dataHeaderV2.getRepetition_levels_byte_length() - dataHeaderV2.getDefinition_levels_byte_length(); pagesInChunk.add( new DataPageV2( dataHeaderV2.getNum_rows(), dataHeaderV2.getNum_nulls(), dataHeaderV2.getNum_values(), this.readAsBytesInput(dataHeaderV2.getRepetition_levels_byte_length()), this.readAsBytesInput(dataHeaderV2.getDefinition_levels_byte_length()), converter.getEncoding(dataHeaderV2.getEncoding()), this.readAsBytesInput(dataSize), uncompressedPageSize, converter.fromParquetStatistics( getFileMetaData().getCreatedBy(), dataHeaderV2.getStatistics(), type), dataHeaderV2.isIs_compressed())); valuesCountReadSoFar += dataHeaderV2.getNum_values(); ++dataPageCountReadSoFar; break; default: LOG.debug( "skipping page of type {} of size {}", pageHeader.getType(), compressedPageSize); stream.skipFully(compressedPageSize); break; } } if (offsetIndex == null && valuesCountReadSoFar != descriptor.metadata.getValueCount()) { // Would be nice to have a CorruptParquetFileException or something as a subclass? throw new IOException( "Expected " + descriptor.metadata.getValueCount() + " values in column chunk at " + file + " offset " + descriptor.metadata.getFirstDataPageOffset() + " but got " + valuesCountReadSoFar + " values instead over " + pagesInChunk.size() + " pages ending at file offset " + (descriptor.fileOffset + stream.position())); } CompressionCodecFactory.BytesInputDecompressor decompressor = options.getCodecFactory().getDecompressor(descriptor.metadata.getCodec()); return new ColumnPageReader( decompressor, pagesInChunk, dictionaryPage, offsetIndex, blocks.get(currentBlock).getRowCount(), pageBlockDecryptor, aadPrefix, rowGroupOrdinal, columnOrdinal); } private boolean hasMorePages(long valuesCountReadSoFar, int dataPageCountReadSoFar) { return offsetIndex == null ? valuesCountReadSoFar < descriptor.metadata.getValueCount() : dataPageCountReadSoFar < offsetIndex.getPageCount(); } private int getPageOrdinal(int dataPageCountReadSoFar) { if (null == offsetIndex) { return dataPageCountReadSoFar; } return offsetIndex.getPageOrdinal(dataPageCountReadSoFar); } /** * @param size the size of the page * @return the page * @throws IOException if there is an error while reading from the file stream */ public BytesInput readAsBytesInput(int size) throws IOException { return BytesInput.from(stream.sliceBuffers(size)); } } /** * Describes a list of consecutive parts to be read at once. A consecutive part may contain whole * column chunks or only parts of them (some pages). */ private class ConsecutivePartList { private final long offset; private final List chunks = new ArrayList<>(); private long length; private final SQLMetric fileReadTimeMetric; private final SQLMetric fileReadSizeMetric; private final SQLMetric readThroughput; List buffers; /** * Constructor * * @param offset where the first chunk starts */ ConsecutivePartList(long offset) { if (metrics != null) { this.fileReadTimeMetric = metrics.get("ParquetInputFileReadTime"); this.fileReadSizeMetric = metrics.get("ParquetInputFileReadSize"); this.readThroughput = metrics.get("ParquetInputFileReadThroughput"); } else { this.fileReadTimeMetric = null; this.fileReadSizeMetric = null; this.readThroughput = null; } this.offset = offset; } /** * Adds a chunk to the list. It must be consecutive to the previous chunk. * * @param descriptor a chunk descriptor */ public void addChunk(ChunkDescriptor descriptor) { chunks.add(descriptor); length += descriptor.size; } private void allocateReadBuffers() { int fullAllocations = Math.toIntExact(length / options.getMaxAllocationSize()); int lastAllocationSize = Math.toIntExact(length % options.getMaxAllocationSize()); int numAllocations = fullAllocations + (lastAllocationSize > 0 ? 1 : 0); this.buffers = new ArrayList<>(numAllocations); for (int i = 0; i < fullAllocations; i += 1) { this.buffers.add(options.getAllocator().allocate(options.getMaxAllocationSize())); } if (lastAllocationSize > 0) { this.buffers.add(options.getAllocator().allocate(lastAllocationSize)); } } /** * @param f file to read the chunks from * @param builder used to build chunk list to read the pages for the different columns * @throws IOException if there is an error while reading from the stream */ public void readAll(SeekableInputStream f, ChunkListBuilder builder) throws IOException { f.seek(offset); allocateReadBuffers(); long startNs = System.nanoTime(); for (ByteBuffer buffer : buffers) { f.readFully(buffer); buffer.flip(); } setReadMetrics(startNs); // report in a counter the data we just scanned BenchmarkCounter.incrementBytesRead(length); ByteBufferInputStream stream = ByteBufferInputStream.wrap(buffers); for (int i = 0; i < chunks.size(); i++) { ChunkDescriptor descriptor = chunks.get(i); if (descriptor.col != null) { builder.add(descriptor, stream.sliceBuffers(descriptor.size)); } else { stream.skipFully(descriptor.size); } } } private void setReadMetrics(long startNs) { long totalFileReadTimeNs = System.nanoTime() - startNs; double sizeInMb = ((double) length) / (1024 * 1024); double timeInSec = ((double) totalFileReadTimeNs) / 1000_0000_0000L; double throughput = sizeInMb / timeInSec; LOG.debug( "Comet: File Read stats: Length: {} MB, Time: {} secs, throughput: {} MB/sec ", sizeInMb, timeInSec, throughput); if (fileReadTimeMetric != null) { fileReadTimeMetric.add(totalFileReadTimeNs); } if (fileReadSizeMetric != null) { fileReadSizeMetric.add(length); } if (readThroughput != null) { readThroughput.set(throughput); } } /** * End position of the last byte of these chunks * * @return the position following the last byte of these chunks */ public long endPos() { return offset + length; } } /** Information needed to read a column chunk or a part of it. */ private static class ChunkDescriptor { private final ColumnDescriptor col; private final ColumnChunkMetaData metadata; private final long fileOffset; private final long size; /** * @param col column this chunk is part of * @param metadata metadata for the column * @param fileOffset offset in the file where this chunk starts * @param size size of the chunk */ ChunkDescriptor( ColumnDescriptor col, ColumnChunkMetaData metadata, long fileOffset, long size) { this.col = col; this.metadata = metadata; this.fileOffset = fileOffset; this.size = size; } @Override public int hashCode() { return col.hashCode(); } @Override public boolean equals(Object obj) { if (this == obj) { return true; } else if (obj instanceof ChunkDescriptor) { return col.equals(((ChunkDescriptor) obj).col); } else { return false; } } } } ================================================ FILE: common/src/main/java/org/apache/comet/parquet/FooterReader.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.parquet; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.parquet.HadoopReadOptions; import org.apache.parquet.ParquetReadOptions; import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.apache.spark.sql.execution.datasources.PartitionedFile; /** * Copied from Spark's `ParquetFooterReader` in order to avoid shading issue around Parquet. * *

`FooterReader` is a util class which encapsulates the helper methods of reading parquet file * footer. */ public class FooterReader { public static ParquetMetadata readFooter(Configuration configuration, PartitionedFile file) throws IOException, URISyntaxException { long start = file.start(); long length = file.length(); Path filePath = new Path(new URI(file.filePath().toString())); CometInputFile inputFile = CometInputFile.fromPath(filePath, configuration); ParquetReadOptions readOptions = HadoopReadOptions.builder(inputFile.getConf(), inputFile.getPath()) .withRange(start, start + length) .build(); ReadOptions cometReadOptions = ReadOptions.builder(configuration).build(); // Use try-with-resources to ensure fd is closed. try (FileReader fileReader = new FileReader(inputFile, readOptions, cometReadOptions)) { return fileReader.getFooter(); } } } ================================================ FILE: common/src/main/java/org/apache/comet/parquet/IcebergCometNativeBatchReader.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.parquet; import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.execution.metric.SQLMetric; import org.apache.spark.sql.types.StructType; /** * A specialized NativeBatchReader for Iceberg that accepts ParquetMetadata as a Thrift encoded byte * array . This allows Iceberg to pass metadata in serialized form with a two-step initialization * pattern. */ public class IcebergCometNativeBatchReader extends NativeBatchReader { public IcebergCometNativeBatchReader(StructType requiredSchema) { super(); this.sparkSchema = requiredSchema; } /** Initialize the reader using FileInfo instead of PartitionedFile. */ public void init( Configuration conf, FileInfo fileInfo, byte[] parquetMetadataBytes, byte[] nativeFilter, int capacity, StructType dataSchema, boolean isCaseSensitive, boolean useFieldId, boolean ignoreMissingIds, boolean useLegacyDateTimestamp, StructType partitionSchema, InternalRow partitionValues, AbstractColumnReader[] preInitializedReaders, Map metrics) throws Throwable { // Set parent fields this.conf = conf; this.fileInfo = fileInfo; this.footer = new ParquetMetadataSerializer().deserialize(parquetMetadataBytes); this.nativeFilter = nativeFilter; this.capacity = capacity; this.dataSchema = dataSchema; this.isCaseSensitive = isCaseSensitive; this.useFieldId = useFieldId; this.ignoreMissingIds = ignoreMissingIds; this.useLegacyDateTimestamp = useLegacyDateTimestamp; this.partitionSchema = partitionSchema; this.partitionValues = partitionValues; this.preInitializedReaders = preInitializedReaders; this.metrics.clear(); if (metrics != null) { this.metrics.putAll(metrics); } // Call parent init method super.init(); } public StructType getSparkSchema() { return this.sparkSchema; } } ================================================ FILE: common/src/main/java/org/apache/comet/parquet/IndexFilter.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.parquet; import java.util.ArrayList; import java.util.List; import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; import org.apache.parquet.internal.column.columnindex.OffsetIndex; import org.apache.parquet.internal.filter2.columnindex.RowRanges; public class IndexFilter { private final RowRanges rowRanges; private final OffsetIndex offsetIndex; private final long totalRowCount; public IndexFilter(RowRanges rowRanges, OffsetIndex offsetIndex, long totalRowCount) { this.rowRanges = rowRanges; this.offsetIndex = offsetIndex; this.totalRowCount = totalRowCount; } OffsetIndex filterOffsetIndex() { List indexMap = new ArrayList<>(); for (int i = 0, n = offsetIndex.getPageCount(); i < n; ++i) { long from = offsetIndex.getFirstRowIndex(i); if (rowRanges.isOverlapping(from, offsetIndex.getLastRowIndex(i, totalRowCount))) { indexMap.add(i); } } int[] indexArray = new int[indexMap.size()]; for (int i = 0; i < indexArray.length; i++) { indexArray[i] = indexMap.get(i); } return new FilteredOffsetIndex(offsetIndex, indexArray); } List calculateOffsetRanges(OffsetIndex filteredOffsetIndex, ColumnChunkMetaData cm) { List ranges = new ArrayList<>(); long firstPageOffset = offsetIndex.getOffset(0); int n = filteredOffsetIndex.getPageCount(); if (n > 0) { OffsetRange currentRange = null; // Add a range for the dictionary page if required long rowGroupOffset = cm.getStartingPos(); if (rowGroupOffset < firstPageOffset) { currentRange = new OffsetRange(rowGroupOffset, (int) (firstPageOffset - rowGroupOffset)); ranges.add(currentRange); } for (int i = 0; i < n; ++i) { long offset = filteredOffsetIndex.getOffset(i); int length = filteredOffsetIndex.getCompressedPageSize(i); if (currentRange == null || !currentRange.extend(offset, length)) { currentRange = new OffsetRange(offset, length); ranges.add(currentRange); } } } return ranges; } private static class FilteredOffsetIndex implements OffsetIndex { private final OffsetIndex offsetIndex; private final int[] indexMap; private FilteredOffsetIndex(OffsetIndex offsetIndex, int[] indexMap) { this.offsetIndex = offsetIndex; this.indexMap = indexMap; } @Override public int getPageOrdinal(int pageIndex) { return indexMap[pageIndex]; } @Override public int getPageCount() { return indexMap.length; } @Override public long getOffset(int pageIndex) { return offsetIndex.getOffset(indexMap[pageIndex]); } @Override public int getCompressedPageSize(int pageIndex) { return offsetIndex.getCompressedPageSize(indexMap[pageIndex]); } @Override public long getFirstRowIndex(int pageIndex) { return offsetIndex.getFirstRowIndex(indexMap[pageIndex]); } @Override public long getLastRowIndex(int pageIndex, long totalRowCount) { int nextIndex = indexMap[pageIndex] + 1; return (nextIndex >= offsetIndex.getPageCount() ? totalRowCount : offsetIndex.getFirstRowIndex(nextIndex)) - 1; } } static class OffsetRange { final long offset; long length; private OffsetRange(long offset, int length) { this.offset = offset; this.length = length; } private boolean extend(long offset, int length) { if (this.offset + this.length == offset) { this.length += length; return true; } else { return false; } } } } ================================================ FILE: common/src/main/java/org/apache/comet/parquet/LazyColumnReader.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.parquet; import java.io.IOException; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.page.PageReader; import org.apache.spark.sql.types.DataType; import org.apache.comet.CometSchemaImporter; import org.apache.comet.vector.CometLazyVector; import org.apache.comet.vector.CometVector; public class LazyColumnReader extends ColumnReader { // Remember the largest skipped index for sanity checking. private int lastSkippedRowId = Integer.MAX_VALUE; // Track whether the underlying page is drained. private boolean isPageDrained = true; // Leftover number of rows that did not skip in the previous batch. private int numRowsToSkipFromPrevBatch; // The lazy vector being updated. private final CometLazyVector vector; LazyColumnReader( DataType sparkReadType, ColumnDescriptor descriptor, CometSchemaImporter importer, int batchSize, boolean useDecimal128, boolean useLegacyDateTimestamp) { super(sparkReadType, descriptor, importer, batchSize, useDecimal128, useLegacyDateTimestamp); this.batchSize = 0; // the batch size is set later in `readBatch` this.vector = new CometLazyVector(sparkReadType, this, useDecimal128); } @Override public void setPageReader(PageReader pageReader) throws IOException { super.setPageReader(pageReader); lastSkippedRowId = Integer.MAX_VALUE; isPageDrained = true; numRowsToSkipFromPrevBatch = 0; currentNumValues = batchSize; } /** * Lazily read a batch of 'total' rows for this column. The includes: 1) Skip any unused rows from * the previous batch 2) Reset the native columnar batch 3) Reset tracking variables * * @param total the number of rows in the batch. MUST be <= the number of rows available in this * column chunk. */ @Override public void readBatch(int total) { // Before starting a new batch, take care of the remaining rows to skip from the previous batch. tryPageSkip(batchSize); numRowsToSkipFromPrevBatch += batchSize - currentNumValues; // Now first reset the current columnar batch so that it can be used to fill in a new batch // of values. Then, keep reading more data pages (via 'readBatch') until the current batch is // full, or we have read 'total' number of values. Native.resetBatch(nativeHandle); batchSize = total; currentNumValues = 0; lastSkippedRowId = -1; } @Override public CometVector currentBatch() { return vector; } /** Read all rows up to the `batchSize`. Expects no rows are skipped so far. */ public void readAllBatch() { // All rows should be read without any skips so far assert (lastSkippedRowId == -1); readBatch(batchSize - 1, 0); } /** * Read at least up to `rowId`. It may read beyond `rowId` if enough rows available in the page. * It may skip reading rows before `rowId`. In case `rowId` is already read, return immediately. * * @param rowId the row index in the batch to read. * @return true if `rowId` is newly materialized, or false if `rowId` is already materialized. */ public boolean materializeUpToIfNecessary(int rowId) { // Not allowed reading rowId if it may have skipped previously. assert (rowId > lastSkippedRowId); // If `rowId` is already materialized, return immediately. if (rowId < currentNumValues) return false; int numRowsWholePageSkipped = tryPageSkip(rowId); readBatch(rowId, numRowsWholePageSkipped); return true; } /** * Read up to `rowId` (inclusive). If the whole pages are skipped previously in `tryPageSkip()`, * pad the number of whole page skipped rows with nulls to the underlying vector before reading. * * @param rowId the row index in the batch to read. * @param numNullRowsToPad the number of nulls to pad before reading. */ private void readBatch(int rowId, int numNullRowsToPad) { if (numRowsToSkipFromPrevBatch > 0) { // Reaches here only when starting a new batch and the page is previously drained readPage(); isPageDrained = false; Native.skipBatch(nativeHandle, numRowsToSkipFromPrevBatch, true); numRowsToSkipFromPrevBatch = 0; } while (rowId >= currentNumValues) { int numRowsToRead = batchSize - currentNumValues; if (isPageDrained) { readPage(); } int[] array = Native.readBatch(nativeHandle, numRowsToRead, numNullRowsToPad); int read = array[0]; isPageDrained = read < numRowsToRead; currentNumValues += read; currentNumNulls += array[1]; // No need to update numNullRowsToPad. numNullRowsToPad > 0 means there were whole page skips. // That guarantees that the Native.readBatch can read up to rowId in the current page. } } /** * Try to skip until `rowId` (exclusive). If possible, it skips whole underlying pages without * decompressing. In that case, it returns early at the page end, so that the next iteration can * lazily decide to `readPage()` or `tryPageSkip()` again. * * @param rowId the row index in the batch that it tries to skip up until (exclusive). * @return the number of rows that the whole page skips were applied. */ private int tryPageSkip(int rowId) { int total = rowId - currentNumValues; int wholePageSkipped = 0; if (total > 0) { // First try to skip from the non-drained underlying page. int skipped = isPageDrained ? 0 : Native.skipBatch(nativeHandle, total); total -= skipped; isPageDrained = total > 0; if (isPageDrained) { ColumnPageReader columnPageReader = (ColumnPageReader) pageReader; // It is always `columnPageReader.getPageValueCount() > numRowsToSkipFromPriorBatch` int pageValueCount = columnPageReader.getPageValueCount() - numRowsToSkipFromPrevBatch; while (pageValueCount <= total) { // skip the entire page if the next page is small enough columnPageReader.skipPage(); numRowsToSkipFromPrevBatch = 0; total -= pageValueCount; wholePageSkipped += pageValueCount; pageValueCount = columnPageReader.getPageValueCount(); } } currentNumValues += skipped + wholePageSkipped; currentNumNulls += skipped; lastSkippedRowId = currentNumValues - 1; } return wholePageSkipped; } } ================================================ FILE: common/src/main/java/org/apache/comet/parquet/Native.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.parquet; import java.util.Map; import org.apache.comet.IcebergApi; import org.apache.comet.NativeBase; public final class Native extends NativeBase { public static int[] readBatch(long handle, int batchSize) { return readBatch(handle, batchSize, 0); } public static int skipBatch(long handle, int batchSize) { return skipBatch(handle, batchSize, false); } /** Native APIs * */ /** * Creates a reader for a primitive Parquet column. * * @param physicalTypeId id for Parquet physical type * @param logicalTypeId id for Parquet logical type * @param expectedPhysicalTypeId id for Parquet physical type, converted from Spark read type. * This is used for type promotion. * @param path the path from the root schema to the column, derived from the method * 'ColumnDescriptor#getPath()'. * @param maxDl the maximum definition level of the primitive column * @param maxRl the maximum repetition level of the primitive column * @param bitWidth (only set when logical type is INT) the bit width for the integer type (INT8, * INT16, INT32, etc) * @param isSigned (only set when logical type is INT) whether it is signed or unsigned int. * @param typeLength number of bytes required to store a value of the type, only set when the * physical type is FIXED_LEN_BYTE_ARRAY, otherwise it's 0. * @param precision (only set when logical type is DECIMAL) precision of the decimal type * @param expectedPrecision (only set when logical type is DECIMAL) precision of the decimal type * from Spark read schema. This is used for type promotion. * @param scale (only set when logical type is DECIMAL) scale of the decimal type * @param tu (only set when logical type is TIMESTAMP) unit for the timestamp * @param isAdjustedUtc (only set when logical type is TIMESTAMP) whether the timestamp is * adjusted to UTC or not * @param batchSize the batch size for the columnar read * @param useDecimal128 whether to always return 128 bit decimal regardless of precision * @param useLegacyDateTimestampOrNTZ whether to read legacy dates/timestamps as it is * @return a pointer to a native Parquet column reader created */ public static native long initColumnReader( int physicalTypeId, int logicalTypeId, int expectedPhysicalTypeId, String[] path, int maxDl, int maxRl, int bitWidth, int expectedBitWidth, boolean isSigned, int typeLength, int precision, int expectedPrecision, int scale, int expectedScale, int tu, boolean isAdjustedUtc, int batchSize, boolean useDecimal128, boolean useLegacyDateTimestampOrNTZ); /** * Pass a Parquet dictionary page to the native column reader. Note this should only be called * once per Parquet column chunk. Otherwise it'll panic. * * @param handle the handle to the native Parquet column reader * @param dictionaryValueCount the number of values in this dictionary * @param dictionaryData the actual dictionary page data, including repetition/definition levels * as well as values * @param encoding the encoding used by the dictionary */ public static native void setDictionaryPage( long handle, int dictionaryValueCount, byte[] dictionaryData, int encoding); /** * Passes a Parquet data page V1 to the native column reader. * * @param handle the handle to the native Parquet column reader * @param pageValueCount the number of values in this data page * @param pageData the actual page data, which should only contain PLAIN-encoded values. * @param valueEncoding the encoding used by the values */ public static native void setPageV1( long handle, int pageValueCount, byte[] pageData, int valueEncoding); /** * Passes a Parquet data page V2 to the native column reader. * * @param handle the handle to the native Parquet column reader * @param pageValueCount the number of values in this data page * @param defLevelData the data for definition levels * @param repLevelData the data for repetition levels * @param valueData the data for values * @param valueEncoding the encoding used by the values */ public static native void setPageV2( long handle, int pageValueCount, byte[] defLevelData, byte[] repLevelData, byte[] valueData, int valueEncoding); /** * Reset the current columnar batch. This will clear all the content of the batch as well as any * internal state such as the current offset. * * @param handle the handle to the native Parquet column reader */ @IcebergApi public static native void resetBatch(long handle); /** * Reads at most 'batchSize' number of rows from the native Parquet column reader. Returns a tuple * where the first element is the actual number of rows read (including both nulls and non-nulls), * and the second element is the number of nulls read. * *

If the returned value is < 'batchSize' then it means the current page has been completely * drained. In this case, the caller should call {@link Native#setPageV1} or {@link * Native#setPageV2} before the next 'readBatch' call. * *

Note that the current page could also be drained if the returned value = 'batchSize', i.e., * the remaining number of rows in the page is exactly equal to 'batchSize'. In this case, the * next 'readBatch' call will return 0 and the caller should call {@link Native#setPageV1} or * {@link Native#setPageV2} next. * *

If `nullPadSize` > 0, it pads nulls into the underlying vector before the values will be * read into. * * @param handle the handle to the native Parquet column reader * @param batchSize the number of rows to be read * @param nullPadSize the number of nulls to pad before reading. * @return a tuple: (the actual number of rows read, the number of nulls read) */ public static native int[] readBatch(long handle, int batchSize, int nullPadSize); /** * Skips at most 'batchSize' number of rows from the native Parquet column reader, and returns the * actual number of rows skipped. * *

If the returned value is < 'batchSize' then it means the current page has been completely * drained. In this case, the caller should call {@link Native#setPageV1} or {@link * Native#setPageV2} before the next 'skipBatch' call. * *

Note that the current page could also be drained if the returned value = 'batchSize', i.e., * the remaining number of rows in the page is exactly equal to 'batchSize'. In this case, the * next 'skipBatch' call will return 0 and the caller should call {@link Native#setPageV1} or * {@link Native#setPageV2} next. * * @param handle the handle to the native Parquet column reader * @param batchSize the number of rows to skip in the current page * @param discard if true, discard read rows without padding nulls into the underlying vector * @return the actual number of rows skipped */ public static native int skipBatch(long handle, int batchSize, boolean discard); /** * Returns the current batch constructed via 'readBatch' * * @param handle the handle to the native Parquet column reader * @param arrayAddr the memory address to the ArrowArray struct * @param schemaAddr the memory address to the ArrowSchema struct */ public static native void currentBatch(long handle, long arrayAddr, long schemaAddr); /** * Closes the native Parquet column reader and releases all resources associated with it. * * @param handle the handle to the native Parquet column reader */ public static native void closeColumnReader(long handle); ///////////// Arrow Native Parquet Reader APIs // TODO: Add partitionValues(?), improve requiredColumns to use a projection mask that corresponds // to arrow. // Add batch size, datetimeRebaseModeSpec, metrics(how?)... /** * Verify that object store options are valid. An exception will be thrown if the provided options * are not valid. */ public static native void validateObjectStoreConfig( String filePath, Map objectStoreOptions); /** * Initialize a record batch reader for a PartitionedFile * * @param filePath * @param starts * @param lengths * @return a handle to the record batch reader, used in subsequent calls. */ public static native long initRecordBatchReader( String filePath, long fileSize, long[] starts, long[] lengths, byte[] filter, byte[] requiredSchema, byte[] dataSchema, String sessionTimezone, int batchSize, boolean caseSensitive, Map objectStoreOptions, CometFileKeyUnwrapper keyUnwrapper, Object metricsNode); // arrow native version of read batch /** * Read the next batch of data into memory on native side * * @param handle * @return the number of rows read */ public static native int readNextRecordBatch(long handle); // arrow native equivalent of currentBatch. 'columnNum' is number of the column in the record // batch /** * Load the column corresponding to columnNum in the currently loaded record batch into JVM * * @param handle * @param columnNum * @param arrayAddr * @param schemaAddr */ public static native void currentColumnBatch( long handle, int columnNum, long arrayAddr, long schemaAddr); // arrow native version to close record batch reader /** * Close the record batch reader. Free the resources * * @param handle */ public static native void closeRecordBatchReader(long handle); } ================================================ FILE: common/src/main/java/org/apache/comet/parquet/NativeBatchReader.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.parquet; import java.io.ByteArrayOutputStream; import java.io.Closeable; import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.URI; import java.net.URISyntaxException; import java.nio.channels.Channels; import java.util.*; import java.util.stream.Collectors; import scala.Option; import scala.collection.Seq; import scala.collection.mutable.Buffer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.ipc.WriteChannel; import org.apache.arrow.vector.ipc.message.MessageSerializer; import org.apache.arrow.vector.types.pojo.Schema; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.RecordReader; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.parquet.HadoopReadOptions; import org.apache.parquet.ParquetReadOptions; import org.apache.parquet.Preconditions; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.hadoop.metadata.BlockMetaData; import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.Type; import org.apache.spark.TaskContext; import org.apache.spark.TaskContext$; import org.apache.spark.executor.TaskMetrics; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.comet.parquet.CometParquetReadSupport; import org.apache.spark.sql.comet.util.Utils$; import org.apache.spark.sql.errors.QueryExecutionErrors; import org.apache.spark.sql.execution.datasources.PartitionedFile; import org.apache.spark.sql.execution.datasources.parquet.ParquetColumn; import org.apache.spark.sql.execution.datasources.parquet.ParquetToSparkSchemaConverter; import org.apache.spark.sql.execution.datasources.parquet.ParquetUtils; import org.apache.spark.sql.execution.metric.SQLMetric; import org.apache.spark.sql.internal.SQLConf; import org.apache.spark.sql.types.*; import org.apache.spark.sql.vectorized.ColumnarBatch; import org.apache.spark.util.AccumulatorV2; import org.apache.comet.CometConf; import org.apache.comet.CometSchemaImporter; import org.apache.comet.objectstore.NativeConfig; import org.apache.comet.shims.ShimBatchReader; import org.apache.comet.shims.ShimFileFormat; import org.apache.comet.vector.CometVector; import org.apache.comet.vector.NativeUtil; import static scala.jdk.javaapi.CollectionConverters.asJava; /** * A vectorized Parquet reader that reads a Parquet file in a batched fashion. * *

Example of how to use this: * * 

 *   NativeBatchReader reader = new NativeBatchReader(parquetFile, batchSize);
 *   try {
 *     reader.init();
 *     while (reader.readBatch()) {
 *       ColumnarBatch batch = reader.currentBatch();
 *       // consume the batch
 *     }
 *   } finally { // resources associated with the reader should be released
 *     reader.close();
 *   }
 *
*/ public class NativeBatchReader extends RecordReader implements Closeable { /** * A class that contains the necessary file information for reading a Parquet file. This class * provides an abstraction over PartitionedFile properties. */ public static class FileInfo { private final long start; private final long length; private final String filePath; private final long fileSize; public FileInfo(long start, long length, String filePath, long fileSize) throws URISyntaxException { this.start = start; this.length = length; URI uri = new Path(filePath).toUri(); if (uri.getScheme() == null) { uri = new Path("file://" + filePath).toUri(); } this.filePath = uri.toString(); this.fileSize = fileSize; } public static FileInfo fromPartitionedFile(PartitionedFile file) throws URISyntaxException { return new FileInfo(file.start(), file.length(), file.filePath().toString(), file.fileSize()); } public long start() { return start; } public long length() { return length; } public String filePath() { return filePath; } public long fileSize() { return fileSize; } public URI pathUri() throws URISyntaxException { return new URI(filePath); } } private static final Logger LOG = LoggerFactory.getLogger(NativeBatchReader.class); protected static final BufferAllocator ALLOCATOR = new RootAllocator(); private NativeUtil nativeUtil = new NativeUtil(); protected Configuration conf; protected int capacity; protected boolean isCaseSensitive; protected boolean useFieldId; protected boolean ignoreMissingIds; protected StructType partitionSchema; protected InternalRow partitionValues; protected PartitionedFile file; protected FileInfo fileInfo; protected final Map metrics; // Unfortunately CometMetricNode is from the "spark" package and cannot be used directly here // TODO: Move it to common package? protected Object metricsNode = null; protected StructType sparkSchema; protected StructType dataSchema; MessageType fileSchema; protected MessageType requestedSchema; protected CometVector[] vectors; protected AbstractColumnReader[] columnReaders; protected CometSchemaImporter importer; protected ColumnarBatch currentBatch; // private FileReader fileReader; protected boolean[] missingColumns; protected boolean isInitialized; protected ParquetMetadata footer; protected byte[] nativeFilter; protected AbstractColumnReader[] preInitializedReaders; private ParquetColumn parquetColumn; /** * Map from field name to spark schema index for efficient lookups during batch loading. Built * once during initialization and reused across all batch loads. */ private Map sparkFieldIndexMap; /** * Whether the native scan should always return decimal represented by 128 bits, regardless of its * precision. Normally, this should be true if native execution is enabled, since Arrow compute * kernels doesn't support 32 and 64 bit decimals yet. */ // TODO: (ARROW NATIVE) private boolean useDecimal128; /** * Whether to return dates/timestamps that were written with legacy hybrid (Julian + Gregorian) * calendar as it is. If this is true, Comet will return them as it is, instead of rebasing them * to the new Proleptic Gregorian calendar. If this is false, Comet will throw exceptions when * seeing these dates/timestamps. */ // TODO: (ARROW NATIVE) protected boolean useLegacyDateTimestamp; /** The TaskContext object for executing this task. */ private final TaskContext taskContext; private long totalRowCount = 0; private long handle; // Protected no-arg constructor for subclasses protected NativeBatchReader() { this.taskContext = TaskContext$.MODULE$.get(); this.metrics = new HashMap<>(); } // Only for testing public NativeBatchReader(String file, int capacity) { this(file, capacity, null, null); } // Only for testing public NativeBatchReader( String file, int capacity, StructType partitionSchema, InternalRow partitionValues) { this(new Configuration(), file, capacity, partitionSchema, partitionValues); } // Only for testing public NativeBatchReader( Configuration conf, String file, int capacity, StructType partitionSchema, InternalRow partitionValues) { this.conf = conf; this.capacity = capacity; this.isCaseSensitive = false; this.useFieldId = false; this.ignoreMissingIds = false; this.partitionSchema = partitionSchema; this.partitionValues = partitionValues; this.file = ShimBatchReader.newPartitionedFile(partitionValues, file); this.metrics = new HashMap<>(); this.taskContext = TaskContext$.MODULE$.get(); } private NativeBatchReader(AbstractColumnReader[] columnReaders) { // Todo: set useDecimal128 and useLazyMaterialization int numColumns = columnReaders.length; this.columnReaders = new AbstractColumnReader[numColumns]; vectors = new CometVector[numColumns]; currentBatch = new ColumnarBatch(vectors); // This constructor is used by Iceberg only. The columnReaders are // initialized in Iceberg, so no need to call the init() isInitialized = true; this.taskContext = TaskContext$.MODULE$.get(); this.metrics = new HashMap<>(); } NativeBatchReader( Configuration conf, PartitionedFile inputSplit, ParquetMetadata footer, byte[] nativeFilter, int capacity, StructType sparkSchema, StructType dataSchema, boolean isCaseSensitive, boolean useFieldId, boolean ignoreMissingIds, boolean useLegacyDateTimestamp, StructType partitionSchema, InternalRow partitionValues, Map metrics, Object metricsNode) { this.conf = conf; this.capacity = capacity; this.sparkSchema = sparkSchema; this.dataSchema = dataSchema; this.isCaseSensitive = isCaseSensitive; this.useFieldId = useFieldId; this.ignoreMissingIds = ignoreMissingIds; this.useLegacyDateTimestamp = useLegacyDateTimestamp; this.partitionSchema = partitionSchema; this.partitionValues = partitionValues; this.file = inputSplit; this.footer = footer; this.nativeFilter = nativeFilter; this.metrics = metrics; this.metricsNode = metricsNode; this.taskContext = TaskContext$.MODULE$.get(); } /** Alternate constructor that accepts FileInfo instead of PartitionedFile. */ NativeBatchReader( Configuration conf, FileInfo fileInfo, ParquetMetadata footer, byte[] nativeFilter, int capacity, StructType sparkSchema, StructType dataSchema, boolean isCaseSensitive, boolean useFieldId, boolean ignoreMissingIds, boolean useLegacyDateTimestamp, StructType partitionSchema, InternalRow partitionValues, Map metrics, Object metricsNode) { this.conf = conf; this.capacity = capacity; this.sparkSchema = sparkSchema; this.dataSchema = dataSchema; this.isCaseSensitive = isCaseSensitive; this.useFieldId = useFieldId; this.ignoreMissingIds = ignoreMissingIds; this.useLegacyDateTimestamp = useLegacyDateTimestamp; this.partitionSchema = partitionSchema; this.partitionValues = partitionValues; this.fileInfo = fileInfo; this.footer = footer; this.nativeFilter = nativeFilter; this.metrics = metrics; this.metricsNode = metricsNode; this.taskContext = TaskContext$.MODULE$.get(); } /** * Initialize this reader. The reason we don't do it in the constructor is that we want to close * any resource hold by this reader when error happens during the initialization. */ public void init() throws Throwable { useDecimal128 = conf.getBoolean( CometConf.COMET_USE_DECIMAL_128().key(), (Boolean) CometConf.COMET_USE_DECIMAL_128().defaultValue().get()); // Use fileInfo if available, otherwise fall back to file long start = fileInfo != null ? fileInfo.start() : file.start(); long length = fileInfo != null ? fileInfo.length() : file.length(); String filePath = fileInfo != null ? fileInfo.filePath() : file.filePath().toString(); long fileSize = fileInfo != null ? fileInfo.fileSize() : file.fileSize(); URI pathUri = fileInfo != null ? fileInfo.pathUri() : file.pathUri(); ParquetReadOptions.Builder builder = HadoopReadOptions.builder(conf, new Path(filePath)); if (start >= 0 && length >= 0) { builder = builder.withRange(start, start + length); } ParquetReadOptions readOptions = builder.build(); Map objectStoreOptions = asJava(NativeConfig.extractObjectStoreOptions(conf, pathUri)); // TODO: enable off-heap buffer when they are ready ReadOptions cometReadOptions = ReadOptions.builder(conf).build(); Path path = new Path(new URI(filePath)); try (FileReader fileReader = new FileReader( CometInputFile.fromPath(path, conf), footer, readOptions, cometReadOptions, metrics)) { requestedSchema = footer.getFileMetaData().getSchema(); fileSchema = requestedSchema; if (sparkSchema == null) { ParquetToSparkSchemaConverter converter = new ParquetToSparkSchemaConverter(conf); sparkSchema = converter.convert(requestedSchema); } else { requestedSchema = CometParquetReadSupport.clipParquetSchema( requestedSchema, sparkSchema, isCaseSensitive, useFieldId, ignoreMissingIds); if (requestedSchema.getFieldCount() != sparkSchema.size()) { throw new IllegalArgumentException( String.format( "Spark schema has %d columns while " + "Parquet schema has %d columns", sparkSchema.size(), requestedSchema.getFieldCount())); } } boolean caseSensitive = conf.getBoolean( SQLConf.CASE_SENSITIVE().key(), (boolean) SQLConf.CASE_SENSITIVE().defaultValue().get()); // rename spark fields based on field_id so name of spark schema field matches the parquet // field name if (useFieldId && ParquetUtils.hasFieldIds(sparkSchema)) { sparkSchema = getSparkSchemaByFieldId(sparkSchema, requestedSchema.asGroupType(), caseSensitive); } this.parquetColumn = getParquetColumn(requestedSchema, this.sparkSchema); // Create Column readers List fields = requestedSchema.getFields(); List fileFields = fileSchema.getFields(); ParquetColumn[] parquetFields = asJava(parquetColumn.children()).toArray(new ParquetColumn[0]); int numColumns = fields.size(); if (partitionSchema != null) numColumns += partitionSchema.size(); columnReaders = new AbstractColumnReader[numColumns]; // Initialize missing columns and use null vectors for them missingColumns = new boolean[numColumns]; // We do not need the column index of the row index; but this method has the // side effect of throwing an exception if a column with the same name is // found which we do want (spark unit tests explicitly test for that). ShimFileFormat.findRowIndexColumnIndexInSchema(sparkSchema); StructField[] nonPartitionFields = sparkSchema.fields(); boolean hasRowIndexColumn = false; // Ranges of rows to read (needed iff row indexes are being read) List blocks = FileReader.filterRowGroups(readOptions, footer.getBlocks(), fileReader); totalRowCount = fileReader.getFilteredRecordCount(); if (totalRowCount == 0) { // all the data is filtered out. isInitialized = true; return; } long[] starts = new long[blocks.size()]; long[] lengths = new long[blocks.size()]; int blockIndex = 0; for (BlockMetaData block : blocks) { long blockStart = block.getStartingPos(); long blockLength = block.getCompressedSize(); starts[blockIndex] = blockStart; lengths[blockIndex] = blockLength; blockIndex++; } for (int i = 0; i < fields.size(); i++) { Type field = fields.get(i); Optional optFileField = fileFields.stream().filter(f -> f.getName().equals(field.getName())).findFirst(); if (nonPartitionFields[i].name().equals(ShimFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME())) { // Values of ROW_INDEX_TEMPORARY_COLUMN_NAME column are always populated with // generated row indexes, rather than read from the file. // TODO(SPARK-40059): Allow users to include columns named // FileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME in their schemas. long[] rowIndices = FileReader.getRowIndices(blocks); columnReaders[i] = new ArrowRowIndexColumnReader(nonPartitionFields[i], capacity, rowIndices); hasRowIndexColumn = true; missingColumns[i] = true; } else if (optFileField.isPresent()) { // The column we are reading may be a complex type in which case we check if each field in // the requested type is in the file type (and the same data type) // This makes the same check as Spark's VectorizedParquetReader checkColumn(parquetFields[i]); missingColumns[i] = false; } else { if (preInitializedReaders != null && i < preInitializedReaders.length && preInitializedReaders[i] != null) { columnReaders[i] = preInitializedReaders[i]; missingColumns[i] = true; } else { if (field.getRepetition() == Type.Repetition.REQUIRED) { throw new IOException( "Required column '" + field.getName() + "' is missing" + " in data file " + filePath); } if (field.isPrimitive()) { ArrowConstantColumnReader reader = new ArrowConstantColumnReader(nonPartitionFields[i], capacity, useDecimal128); columnReaders[i] = reader; missingColumns[i] = true; } else { // the column requested is not in the file, but the native reader can handle that // and will return nulls for all rows requested missingColumns[i] = false; } } } } // Initialize constant readers for partition columns if (partitionSchema != null) { StructField[] partitionFields = partitionSchema.fields(); for (int i = fields.size(); i < columnReaders.length; i++) { int fieldIndex = i - fields.size(); StructField field = partitionFields[fieldIndex]; ArrowConstantColumnReader reader = new ArrowConstantColumnReader( field, capacity, partitionValues, fieldIndex, useDecimal128); columnReaders[i] = reader; } } vectors = new CometVector[numColumns]; currentBatch = new ColumnarBatch(vectors); // For test purpose only // If the last external accumulator is `NumRowGroupsAccumulator`, the row group number to read // will be updated to the accumulator. So we can check if the row groups are filtered or not // in test case. // Note that this tries to get thread local TaskContext object, if this is called at other // thread, it won't update the accumulator. if (taskContext != null) { Option> accu = getTaskAccumulator(taskContext.taskMetrics()); if (accu.isDefined() && accu.get().getClass().getSimpleName().equals("NumRowGroupsAcc")) { @SuppressWarnings("unchecked") AccumulatorV2 intAccum = (AccumulatorV2) accu.get(); intAccum.add(blocks.size()); } } boolean encryptionEnabled = CometParquetUtils.encryptionEnabled(conf); // Create keyUnwrapper if encryption is enabled CometFileKeyUnwrapper keyUnwrapper = null; if (encryptionEnabled) { keyUnwrapper = new CometFileKeyUnwrapper(); keyUnwrapper.storeDecryptionKeyRetriever(filePath, conf); } // Filter out columns with preinitialized readers from sparkSchema before making the // call to native if (preInitializedReaders != null) { StructType filteredSchema = new StructType(); StructField[] sparkFields = sparkSchema.fields(); // Build name map for efficient lookups Map fileFieldNameMap = caseSensitive ? buildCaseSensitiveNameMap(fileFields) : buildCaseInsensitiveNameMap(fileFields); for (int i = 0; i < sparkFields.length; i++) { // Keep the column if: // 1. It doesn't have a preinitialized reader, OR // 2. It has a preinitialized reader but exists in fileSchema boolean hasPreInitializedReader = i < preInitializedReaders.length && preInitializedReaders[i] != null; String fieldName = caseSensitive ? sparkFields[i].name() : sparkFields[i].name().toLowerCase(Locale.ROOT); boolean existsInFileSchema = fileFieldNameMap.containsKey(fieldName); if (!hasPreInitializedReader || existsInFileSchema) { filteredSchema = filteredSchema.add(sparkFields[i]); } } sparkSchema = filteredSchema; } // Native code uses "UTC" always as the timeZoneId when converting from spark to arrow schema. String timeZoneId = "UTC"; Schema arrowSchema = Utils$.MODULE$.toArrowSchema(sparkSchema, timeZoneId); byte[] serializedRequestedArrowSchema = serializeArrowSchema(arrowSchema); Schema dataArrowSchema = Utils$.MODULE$.toArrowSchema(dataSchema, timeZoneId); byte[] serializedDataArrowSchema = serializeArrowSchema(dataArrowSchema); int batchSize = conf.getInt( CometConf.COMET_BATCH_SIZE().key(), (Integer) CometConf.COMET_BATCH_SIZE().defaultValue().get()); this.handle = Native.initRecordBatchReader( filePath, fileSize, starts, lengths, hasRowIndexColumn ? null : nativeFilter, serializedRequestedArrowSchema, serializedDataArrowSchema, timeZoneId, batchSize, caseSensitive, objectStoreOptions, keyUnwrapper, metricsNode); // Build spark field index map for efficient lookups during batch loading StructField[] sparkFields = sparkSchema.fields(); sparkFieldIndexMap = new HashMap<>(); for (int j = 0; j < sparkFields.length; j++) { String fieldName = caseSensitive ? sparkFields[j].name() : sparkFields[j].name().toLowerCase(Locale.ROOT); sparkFieldIndexMap.put(fieldName, j); } } isInitialized = true; } private ParquetColumn getParquetColumn(MessageType schema, StructType sparkSchema) { // We use a different config from the config that is passed in. // This follows the setting used in Spark's SpecificParquetRecordReaderBase Configuration config = new Configuration(conf); config.setBoolean(SQLConf.PARQUET_BINARY_AS_STRING().key(), false); config.setBoolean(SQLConf.PARQUET_INT96_AS_TIMESTAMP().key(), false); config.setBoolean(SQLConf.CASE_SENSITIVE().key(), false); config.setBoolean(SQLConf.PARQUET_INFER_TIMESTAMP_NTZ_ENABLED().key(), false); config.setBoolean(SQLConf.LEGACY_PARQUET_NANOS_AS_LONG().key(), false); ParquetToSparkSchemaConverter converter = new ParquetToSparkSchemaConverter(config); return converter.convertParquetColumn(schema, Option.apply(sparkSchema)); } private Map> getIdToParquetFieldMap(GroupType type) { return type.getFields().stream() .filter(f -> f.getId() != null) .collect(Collectors.groupingBy(f -> f.getId().intValue())); } private Map> getCaseSensitiveParquetFieldMap(GroupType schema) { return schema.getFields().stream().collect(Collectors.toMap(Type::getName, Arrays::asList)); } private Map> getCaseInsensitiveParquetFieldMap(GroupType schema) { return schema.getFields().stream() .collect(Collectors.groupingBy(f -> f.getName().toLowerCase(Locale.ROOT))); } private Map buildCaseSensitiveNameMap(List types) { return types.stream().collect(Collectors.toMap(Type::getName, t -> t)); } private Map buildCaseInsensitiveNameMap(List types) { return types.stream() .collect(Collectors.toMap(t -> t.getName().toLowerCase(Locale.ROOT), t -> t)); } private Type getMatchingParquetFieldById( StructField f, Map> idToParquetFieldMap, Map> nameToParquetFieldMap, boolean isCaseSensitive) { List matched = null; int fieldId = 0; if (ParquetUtils.hasFieldId(f)) { fieldId = ParquetUtils.getFieldId(f); matched = idToParquetFieldMap.get(fieldId); } else { String fieldName = isCaseSensitive ? f.name() : f.name().toLowerCase(Locale.ROOT); matched = nameToParquetFieldMap.get(fieldName); } if (matched == null || matched.isEmpty()) { return null; } if (matched.size() > 1) { // Need to fail if there is ambiguity, i.e. more than one field is matched String parquetTypesString = matched.stream().map(Type::getName).collect(Collectors.joining("[", ", ", "]")); throw QueryExecutionErrors.foundDuplicateFieldInFieldIdLookupModeError( fieldId, parquetTypesString); } else { return matched.get(0); } } // Derived from CometParquetReadSupport.matchFieldId private String getMatchingNameById( StructField f, Map> idToParquetFieldMap, Map> nameToParquetFieldMap, boolean isCaseSensitive) { Type matched = getMatchingParquetFieldById(f, idToParquetFieldMap, nameToParquetFieldMap, isCaseSensitive); // When there is no ID match, we use a fake name to avoid a name match by accident // We need this name to be unique as well, otherwise there will be type conflicts if (matched == null) { return CometParquetReadSupport.generateFakeColumnName(); } else { return matched.getName(); } } // clip ParquetGroup Type private StructType getSparkSchemaByFieldId( StructType schema, GroupType parquetSchema, boolean caseSensitive) { StructType newSchema = new StructType(); Map> idToParquetFieldMap = getIdToParquetFieldMap(parquetSchema); Map> nameToParquetFieldMap = caseSensitive ? getCaseSensitiveParquetFieldMap(parquetSchema) : getCaseInsensitiveParquetFieldMap(parquetSchema); for (StructField f : schema.fields()) { DataType newDataType; String fieldName = isCaseSensitive ? f.name() : f.name().toLowerCase(Locale.ROOT); List parquetFieldList = nameToParquetFieldMap.get(fieldName); if (parquetFieldList == null) { newDataType = f.dataType(); } else { Type fieldType = parquetFieldList.get(0); if (f.dataType() instanceof StructType) { newDataType = getSparkSchemaByFieldId( (StructType) f.dataType(), fieldType.asGroupType(), caseSensitive); } else { newDataType = getSparkTypeByFieldId(f.dataType(), fieldType, caseSensitive); } } String matchedName = getMatchingNameById(f, idToParquetFieldMap, nameToParquetFieldMap, isCaseSensitive); StructField newField = f.copy(matchedName, newDataType, f.nullable(), f.metadata()); newSchema = newSchema.add(newField); } return newSchema; } private static boolean isPrimitiveCatalystType(DataType dataType) { return !(dataType instanceof ArrayType) && !(dataType instanceof MapType) && !(dataType instanceof StructType); } private DataType getSparkTypeByFieldId( DataType dataType, Type parquetType, boolean caseSensitive) { DataType newDataType; if (dataType instanceof StructType) { newDataType = getSparkSchemaByFieldId((StructType) dataType, parquetType.asGroupType(), caseSensitive); } else if (dataType instanceof ArrayType && !isPrimitiveCatalystType(((ArrayType) dataType).elementType())) { newDataType = getSparkArrayTypeByFieldId( (ArrayType) dataType, parquetType.asGroupType(), caseSensitive); } else if (dataType instanceof MapType) { MapType mapType = (MapType) dataType; DataType keyType = mapType.keyType(); DataType valueType = mapType.valueType(); DataType newKeyType; DataType newValueType; Type parquetMapType = parquetType.asGroupType().getFields().get(0); Type parquetKeyType = parquetMapType.asGroupType().getType("key"); Type parquetValueType = parquetMapType.asGroupType().getType("value"); if (keyType instanceof StructType) { newKeyType = getSparkSchemaByFieldId( (StructType) keyType, parquetKeyType.asGroupType(), caseSensitive); } else { newKeyType = keyType; } if (valueType instanceof StructType) { newValueType = getSparkSchemaByFieldId( (StructType) valueType, parquetValueType.asGroupType(), caseSensitive); } else { newValueType = valueType; } newDataType = new MapType(newKeyType, newValueType, mapType.valueContainsNull()); } else { newDataType = dataType; } return newDataType; } private DataType getSparkArrayTypeByFieldId( ArrayType arrayType, GroupType parquetList, boolean caseSensitive) { DataType newDataType; DataType elementType = arrayType.elementType(); DataType newElementType; Type parquetElementType; if (parquetList.getLogicalTypeAnnotation() == null && parquetList.isRepetition(Type.Repetition.REPEATED)) { parquetElementType = parquetList; } else { // we expect only non-primitive types here (see clipParquetListTypes for related logic) GroupType repeatedGroup = parquetList.asGroupType().getType(0).asGroupType(); if (repeatedGroup.getFieldCount() > 1 || Objects.equals(repeatedGroup.getName(), "array") || Objects.equals(repeatedGroup.getName(), parquetList.getName() + "_tuple")) { parquetElementType = repeatedGroup; } else { parquetElementType = repeatedGroup.getType(0); } } if (elementType instanceof StructType) { newElementType = getSparkSchemaByFieldId( (StructType) elementType, parquetElementType.asGroupType(), caseSensitive); } else { newElementType = getSparkTypeByFieldId(elementType, parquetElementType, caseSensitive); } newDataType = new ArrayType(newElementType, arrayType.containsNull()); return newDataType; } private void checkParquetType(ParquetColumn column) throws IOException { String[] path = asJava(column.path()).toArray(new String[0]); if (containsPath(fileSchema, path)) { if (column.isPrimitive()) { ColumnDescriptor desc = column.descriptor().get(); ColumnDescriptor fd = fileSchema.getColumnDescription(desc.getPath()); TypeUtil.checkParquetType(fd, column.sparkType()); } else { for (ParquetColumn childColumn : asJava(column.children())) { checkColumn(childColumn); } } } else { // A missing column which is either primitive or complex if (column.required()) { // check if we have a preinitialized column reader for this column. int columnIndex = getColumnIndexFromParquetColumn(column); if (columnIndex == -1 || preInitializedReaders == null || columnIndex >= preInitializedReaders.length || preInitializedReaders[columnIndex] == null) { // Column is missing in data but the required data is non-nullable. This file is invalid. throw new IOException( "Required column is missing in data file. Col: " + Arrays.toString(path)); } } } } /** * Get the column index in the requested schema for a given ParquetColumn. Returns -1 if not * found. */ private int getColumnIndexFromParquetColumn(ParquetColumn column) { String[] targetPath = asJava(column.path()).toArray(new String[0]); if (targetPath.length == 0) { return -1; } // For top-level columns, match by name String columnName = targetPath[0]; ParquetColumn[] parquetFields = asJava(parquetColumn.children()).toArray(new ParquetColumn[0]); for (int i = 0; i < parquetFields.length; i++) { String[] fieldPath = asJava(parquetFields[i].path()).toArray(new String[0]); if (fieldPath.length > 0 && fieldPath[0].equals(columnName)) { return i; } } return -1; } /** * Checks whether the given 'path' exists in 'parquetType'. The difference between this and {@link * MessageType#containsPath(String[])} is that the latter only support paths to leaf From Spark: * VectorizedParquetRecordReader Check whether a column from requested schema is missing from the * file schema, or whether it conforms to the type of the file schema. */ private void checkColumn(ParquetColumn column) throws IOException { String[] path = asJava(column.path()).toArray(new String[0]); if (containsPath(fileSchema, path)) { if (column.isPrimitive()) { ColumnDescriptor desc = column.descriptor().get(); ColumnDescriptor fd = fileSchema.getColumnDescription(desc.getPath()); if (!fd.equals(desc)) { throw new UnsupportedOperationException("Schema evolution not supported."); } } else { for (ParquetColumn childColumn : asJava(column.children())) { checkColumn(childColumn); } } } else { // A missing column which is either primitive or complex if (column.required()) { // Column is missing in data but the required data is non-nullable. This file is invalid. throw new IOException( "Required column is missing in data file. Col: " + Arrays.toString(path)); } } } /** * Checks whether the given 'path' exists in 'parquetType'. The difference between this and {@link * MessageType#containsPath(String[])} is that the latter only support paths to leaf nodes, while * this support paths both to leaf and non-leaf nodes. */ private boolean containsPath(Type parquetType, String[] path) { return containsPath(parquetType, path, 0); } private boolean containsPath(Type parquetType, String[] path, int depth) { if (path.length == depth) return true; if (parquetType instanceof GroupType) { String fieldName = path[depth]; GroupType parquetGroupType = (GroupType) parquetType; if (parquetGroupType.containsField(fieldName)) { return containsPath(parquetGroupType.getType(fieldName), path, depth + 1); } } return false; } public void setSparkSchema(StructType schema) { this.sparkSchema = schema; } public AbstractColumnReader[] getColumnReaders() { return columnReaders; } @Override public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { // Do nothing. The initialization work is done in 'init' already. } @Override public boolean nextKeyValue() throws IOException { return nextBatch(); } @Override public Void getCurrentKey() { return null; } @Override public ColumnarBatch getCurrentValue() { return currentBatch(); } @Override public float getProgress() { return 0; } /** * Returns the current columnar batch being read. * *

Note that this must be called AFTER {@link NativeBatchReader#nextBatch()}. */ public ColumnarBatch currentBatch() { return currentBatch; } /** * Loads the next batch of rows. This is called by Spark _and_ Iceberg * * @return true if there are no more rows to read, false otherwise. */ public boolean nextBatch() throws IOException { Preconditions.checkState(isInitialized, "init() should be called first!"); // if (rowsRead >= totalRowCount) return false; if (totalRowCount == 0) return false; int batchSize; try { batchSize = loadNextBatch(); } catch (RuntimeException e) { // Spark will check certain exception e.g. `SchemaColumnConvertNotSupportedException`. throw e; } catch (Throwable e) { throw new IOException(e); } if (batchSize == 0) return false; long totalDecodeTime = 0, totalLoadTime = 0; for (int i = 0; i < columnReaders.length; i++) { AbstractColumnReader reader = columnReaders[i]; long startNs = System.nanoTime(); // TODO: read from native reader reader.readBatch(batchSize); // totalDecodeTime += System.nanoTime() - startNs; // startNs = System.nanoTime(); vectors[i] = reader.currentBatch(); totalLoadTime += System.nanoTime() - startNs; } // TODO: (ARROW NATIVE) Add Metrics // SQLMetric decodeMetric = metrics.get("ParquetNativeDecodeTime"); // if (decodeMetric != null) { // decodeMetric.add(totalDecodeTime); // } SQLMetric loadMetric = metrics.get("ParquetNativeLoadTime"); if (loadMetric != null) { loadMetric.add(totalLoadTime); } currentBatch.setNumRows(batchSize); return true; } @Override public void close() throws IOException { if (columnReaders != null) { for (AbstractColumnReader reader : columnReaders) { if (reader != null) { reader.close(); } } } if (importer != null) { importer.close(); importer = null; } nativeUtil.close(); if (this.handle > 0) { Native.closeRecordBatchReader(this.handle); this.handle = 0; } } @SuppressWarnings("deprecation") private int loadNextBatch() throws Throwable { for (ParquetColumn childColumn : asJava(parquetColumn.children())) { checkParquetType(childColumn); } int batchSize = Native.readNextRecordBatch(this.handle); if (batchSize == 0) { return batchSize; } if (importer != null) importer.close(); importer = new CometSchemaImporter(ALLOCATOR); List fields = requestedSchema.getFields(); StructField[] sparkFields = sparkSchema.fields(); boolean caseSensitive = conf.getBoolean( SQLConf.CASE_SENSITIVE().key(), (boolean) SQLConf.CASE_SENSITIVE().defaultValue().get()); for (int i = 0; i < fields.size(); i++) { if (!missingColumns[i]) { if (columnReaders[i] != null) columnReaders[i].close(); // TODO: (ARROW NATIVE) handle tz, datetime & int96 rebase Type field = fields.get(i); // Find the corresponding spark field by matching field names using the prebuilt map String fieldName = caseSensitive ? field.getName() : field.getName().toLowerCase(Locale.ROOT); Integer sparkSchemaIndex = sparkFieldIndexMap.get(fieldName); if (sparkSchemaIndex == null) { throw new IOException( "Could not find matching Spark field for Parquet field: " + field.getName()); } DataType dataType = sparkFields[sparkSchemaIndex].dataType(); NativeColumnReader reader = new NativeColumnReader( this.handle, sparkSchemaIndex, dataType, field, null, importer, nativeUtil, capacity, useDecimal128, useLegacyDateTimestamp); columnReaders[i] = reader; } } return batchSize; } // Signature of externalAccums changed from returning a Buffer to returning a Seq. If comet is // expecting a Buffer but the Spark version returns a Seq or vice versa, we get a // method not found exception. @SuppressWarnings("unchecked") private Option> getTaskAccumulator(TaskMetrics taskMetrics) { Method externalAccumsMethod; try { externalAccumsMethod = TaskMetrics.class.getDeclaredMethod("externalAccums"); externalAccumsMethod.setAccessible(true); String returnType = externalAccumsMethod.getReturnType().getName(); if (returnType.equals("scala.collection.mutable.Buffer")) { return ((Buffer>) externalAccumsMethod.invoke(taskMetrics)) .lastOption(); } else if (returnType.equals("scala.collection.Seq")) { return ((Seq>) externalAccumsMethod.invoke(taskMetrics)).lastOption(); } else { return Option.apply(null); // None } } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) { return Option.apply(null); // None } } private byte[] serializeArrowSchema(Schema schema) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); WriteChannel writeChannel = new WriteChannel(Channels.newChannel(out)); MessageSerializer.serialize(writeChannel, schema); return out.toByteArray(); } } ================================================ FILE: common/src/main/java/org/apache/comet/parquet/NativeColumnReader.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.parquet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.arrow.c.ArrowArray; import org.apache.arrow.c.ArrowSchema; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.schema.Type; import org.apache.spark.sql.types.DataType; import org.apache.comet.CometSchemaImporter; import org.apache.comet.vector.*; import static scala.jdk.javaapi.CollectionConverters.*; // TODO: extend ColumnReader instead of AbstractColumnReader to reduce code duplication public class NativeColumnReader extends AbstractColumnReader { protected static final Logger LOG = LoggerFactory.getLogger(NativeColumnReader.class); protected final BufferAllocator ALLOCATOR = new RootAllocator(); /** * The current Comet vector holding all the values read by this column reader. Owned by this * reader and MUST be closed after use. */ private CometDecodedVector currentVector; /** Dictionary values for this column. Only set if the column is using dictionary encoding. */ protected CometDictionary dictionary; /** * The number of values in the current batch, used when we are skipping importing of Arrow * vectors, in which case we'll simply update the null count of the existing vectors. */ int currentNumValues; /** * Whether the last loaded vector contains any null value. This is used to determine if we can * skip vector reloading. If the flag is false, Arrow C API will skip to import the validity * buffer, and therefore we cannot skip vector reloading. */ boolean hadNull; private final CometSchemaImporter importer; private final NativeUtil nativeUtil; private ArrowArray array = null; private ArrowSchema schema = null; private long nativeBatchHandle = 0xDEADBEEFL; private final int columnNum; NativeColumnReader( long nativeBatchHandle, int columnNum, DataType type, Type fieldType, ColumnDescriptor descriptor, CometSchemaImporter importer, NativeUtil nativeUtil, int batchSize, boolean useDecimal128, boolean useLegacyDateTimestamp) { super(type, fieldType, descriptor, useDecimal128, useLegacyDateTimestamp); assert batchSize > 0 : "Batch size must be positive, found " + batchSize; this.batchSize = batchSize; this.nativeUtil = nativeUtil; this.importer = importer; this.nativeBatchHandle = nativeBatchHandle; this.columnNum = columnNum; initNative(); } @Override // Override in order to avoid creation of JVM side column readers protected void initNative() { LOG.debug( "Native column reader {} is initialized", String.join(".", this.type.catalogString())); nativeHandle = 0; } @Override public void readBatch(int total) { LOG.debug("Reading column batch of size = {}", total); this.currentNumValues = total; } /** Returns the {@link CometVector} read by this reader. */ @Override public CometVector currentBatch() { return loadVector(); } @Override public void close() { if (currentVector != null) { currentVector.close(); currentVector = null; } super.close(); } /** Returns a decoded {@link CometDecodedVector Comet vector}. */ public CometDecodedVector loadVector() { LOG.debug("Loading vector for next batch"); // Close the previous vector first to release struct memory allocated to import Arrow array & // schema from native side, through the C data interface if (currentVector != null) { currentVector.close(); } // TODO: ARROW NATIVE : Handle Uuid? array = ArrowArray.allocateNew(ALLOCATOR); schema = ArrowSchema.allocateNew(ALLOCATOR); long arrayAddr = array.memoryAddress(); long schemaAddr = schema.memoryAddress(); Native.currentColumnBatch(nativeBatchHandle, columnNum, arrayAddr, schemaAddr); ArrowArray[] arrays = {array}; ArrowSchema[] schemas = {schema}; CometDecodedVector cometVector = (CometDecodedVector) asJava(nativeUtil.importVector(arrays, schemas)).get(0); // Update whether the current vector contains any null values. This is used in the following // batch(s) to determine whether we can skip loading the native vector. hadNull = cometVector.hasNull(); currentVector = cometVector; return currentVector; } } ================================================ FILE: common/src/main/java/org/apache/comet/parquet/ParquetColumnSpec.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.parquet; import java.util.Map; import org.apache.comet.IcebergApi; /** * Parquet ColumnSpec encapsulates the information withing a Parquet ColumnDescriptor. Utility * methods can convert from and to a ColumnDescriptor The only purpose of this class is to allow * passing of Column descriptors between Comet and Iceberg. This is required because Iceberg shades * Parquet, changing the package of Parquet classes and making then incompatible with Comet. */ @IcebergApi public class ParquetColumnSpec { private final int fieldId; private final String[] path; private final String physicalType; private final int typeLength; private final boolean isRepeated; private final int maxDefinitionLevel; private final int maxRepetitionLevel; // Logical type info private String logicalTypeName; private Map logicalTypeParams; @IcebergApi public ParquetColumnSpec( int fieldId, String[] path, String physicalType, int typeLength, boolean isRepeated, int maxDefinitionLevel, int maxRepetitionLevel, String logicalTypeName, Map logicalTypeParams) { this.fieldId = fieldId; this.path = path; this.physicalType = physicalType; this.typeLength = typeLength; this.isRepeated = isRepeated; this.maxDefinitionLevel = maxDefinitionLevel; this.maxRepetitionLevel = maxRepetitionLevel; this.logicalTypeName = logicalTypeName; this.logicalTypeParams = logicalTypeParams; } @IcebergApi public int getFieldId() { return fieldId; } @IcebergApi public String[] getPath() { return path; } @IcebergApi public String getPhysicalType() { return physicalType; } @IcebergApi public int getTypeLength() { return typeLength; } public boolean isRepeated() { return isRepeated; } @IcebergApi public int getMaxRepetitionLevel() { return maxRepetitionLevel; } @IcebergApi public int getMaxDefinitionLevel() { return maxDefinitionLevel; } @IcebergApi public String getLogicalTypeName() { return logicalTypeName; } @IcebergApi public Map getLogicalTypeParams() { return logicalTypeParams; } } ================================================ FILE: common/src/main/java/org/apache/comet/parquet/ParquetMetadataSerializer.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.parquet; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import org.apache.parquet.format.FileMetaData; import org.apache.parquet.format.Util; import org.apache.parquet.format.converter.ParquetMetadataConverter; import org.apache.parquet.hadoop.metadata.ParquetMetadata; /** * Utility class for serializing and deserializing ParquetMetadata instances to/from byte arrays. * This uses the Parquet format's FileMetaData structure and the underlying Thrift compact protocol * for serialization. */ public class ParquetMetadataSerializer { private final ParquetMetadataConverter converter; public ParquetMetadataSerializer() { this.converter = new ParquetMetadataConverter(); } public ParquetMetadataSerializer(ParquetMetadataConverter converter) { this.converter = converter; } /** * Serializes a ParquetMetadata instance to a byte array. * * @param metadata the ParquetMetadata to serialize * @return the serialized byte array * @throws IOException if an error occurs during serialization */ public byte[] serialize(ParquetMetadata metadata) throws IOException { FileMetaData fileMetaData = converter.toParquetMetadata(1, metadata); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); Util.writeFileMetaData(fileMetaData, outputStream); return outputStream.toByteArray(); } /** * Deserializes a byte array back into a ParquetMetadata instance. * * @param bytes the serialized byte array * @return the deserialized ParquetMetadata * @throws IOException if an error occurs during deserialization */ public ParquetMetadata deserialize(byte[] bytes) throws IOException { ByteArrayInputStream inputStream = new ByteArrayInputStream(bytes); FileMetaData fileMetaData = Util.readFileMetaData(inputStream); return converter.fromParquetMetadata(fileMetaData); } } ================================================ FILE: common/src/main/java/org/apache/comet/parquet/ReadOptions.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.parquet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.spark.SparkEnv; import org.apache.spark.launcher.SparkLauncher; import org.apache.comet.CometConf; import org.apache.comet.IcebergApi; /** * Comet specific Parquet related read options. * *

TODO: merge this with {@link org.apache.parquet.HadoopReadOptions} once PARQUET-2203 is done. */ @IcebergApi public class ReadOptions { private static final Logger LOG = LoggerFactory.getLogger(ReadOptions.class); // Max number of concurrent tasks we expect. Used to autoconfigure S3 client connections public static final int S3A_MAX_EXPECTED_PARALLELISM = 32; // defined in hadoop-aws - org.apache.hadoop.fs.s3a.Constants.MAXIMUM_CONNECTIONS public static final String S3A_MAXIMUM_CONNECTIONS = "fs.s3a.connection.maximum"; // default max connections in S3A - org.apache.hadoop.fs.s3a.Constants.DEFAULT_MAXIMUM_CONNECTIONS public static final int S3A_DEFAULT_MAX_HTTP_CONNECTIONS = 96; public static final String S3A_READAHEAD_RANGE = "fs.s3a.readahead.range"; // Default read ahead range in Hadoop is 64K; we increase it to 1 MB public static final long COMET_DEFAULT_READAHEAD_RANGE = 1 * 1024 * 1024; // 1 MB private final boolean parallelIOEnabled; private final int parallelIOThreadPoolSize; private final boolean ioMergeRanges; private final int ioMergeRangesDelta; private final boolean adjustReadRangeSkew; ReadOptions( boolean parallelIOEnabled, int parallelIOThreadPoolSize, boolean ioMergeRanges, int ioMergeRangesDelta, boolean adjustReadRangeSkew) { this.parallelIOEnabled = parallelIOEnabled; this.parallelIOThreadPoolSize = parallelIOThreadPoolSize; this.ioMergeRanges = ioMergeRanges; this.ioMergeRangesDelta = ioMergeRangesDelta; this.adjustReadRangeSkew = adjustReadRangeSkew; } public boolean isParallelIOEnabled() { return this.parallelIOEnabled; } public int parallelIOThreadPoolSize() { return this.parallelIOThreadPoolSize; } public boolean isIOMergeRangesEnabled() { return ioMergeRanges; } public int getIOMergeRangesDelta() { return ioMergeRangesDelta; } public boolean adjustReadRangesSkew() { return adjustReadRangeSkew; } @IcebergApi public static Builder builder(Configuration conf) { return new Builder(conf); } @IcebergApi public static class Builder { private final Configuration conf; private boolean parallelIOEnabled; private int parallelIOThreadPoolSize; private boolean ioMergeRanges; private int ioMergeRangesDelta; private boolean adjustReadRangeSkew; /** * Whether to enable Parquet parallel IO when reading row groups. If true, Parquet reader will * use multiple threads to read multiple chunks of data from the current row group in parallel. */ public Builder enableParallelIO(boolean b) { this.parallelIOEnabled = b; return this; } /** * Specify the number of threads to be used in parallel IO. * *

Note: this will only be effective if parallel IO is enabled (e.g., via {@link * #enableParallelIO(boolean)}). */ public Builder withParallelIOThreadPoolSize(int numThreads) { this.parallelIOThreadPoolSize = numThreads; return this; } public Builder enableIOMergeRanges(boolean enableIOMergeRanges) { this.ioMergeRanges = enableIOMergeRanges; return this; } public Builder withIOMergeRangesDelta(int ioMergeRangesDelta) { this.ioMergeRangesDelta = ioMergeRangesDelta; return this; } public Builder adjustReadRangeSkew(boolean adjustReadRangeSkew) { this.adjustReadRangeSkew = adjustReadRangeSkew; return this; } @IcebergApi public ReadOptions build() { return new ReadOptions( parallelIOEnabled, parallelIOThreadPoolSize, ioMergeRanges, ioMergeRangesDelta, adjustReadRangeSkew); } @IcebergApi public Builder(Configuration conf) { this.conf = conf; this.parallelIOEnabled = conf.getBoolean( CometConf.COMET_PARQUET_PARALLEL_IO_ENABLED().key(), (Boolean) CometConf.COMET_PARQUET_PARALLEL_IO_ENABLED().defaultValue().get()); this.parallelIOThreadPoolSize = conf.getInt( CometConf.COMET_PARQUET_PARALLEL_IO_THREADS().key(), (Integer) CometConf.COMET_PARQUET_PARALLEL_IO_THREADS().defaultValue().get()); this.ioMergeRanges = conf.getBoolean( CometConf.COMET_IO_MERGE_RANGES().key(), (boolean) CometConf.COMET_IO_MERGE_RANGES().defaultValue().get()); this.ioMergeRangesDelta = conf.getInt( CometConf.COMET_IO_MERGE_RANGES_DELTA().key(), (Integer) CometConf.COMET_IO_MERGE_RANGES_DELTA().defaultValue().get()); this.adjustReadRangeSkew = conf.getBoolean( CometConf.COMET_IO_ADJUST_READRANGE_SKEW().key(), (Boolean) CometConf.COMET_IO_ADJUST_READRANGE_SKEW().defaultValue().get()); // override some S3 defaults setS3Config(); } // For paths to S3, if the s3 connection pool max is less than twice the product of // parallel reader threads * number of cores, then increase the connection pool max private void setS3Config() { int s3ConnectionsMax = S3A_DEFAULT_MAX_HTTP_CONNECTIONS; SparkEnv env = SparkEnv.get(); // Use a default number of cores in case we are using the FileReader outside the context // of Spark. int numExecutorCores = S3A_MAX_EXPECTED_PARALLELISM; if (env != null) { numExecutorCores = env.conf().getInt(SparkLauncher.EXECUTOR_CORES, numExecutorCores); } int parallelReaderThreads = this.parallelIOEnabled ? this.parallelIOThreadPoolSize : 1; s3ConnectionsMax = Math.max(numExecutorCores * parallelReaderThreads * 2, s3ConnectionsMax); setS3ConfIfGreater(conf, S3A_MAXIMUM_CONNECTIONS, s3ConnectionsMax); setS3ConfIfGreater(conf, S3A_READAHEAD_RANGE, COMET_DEFAULT_READAHEAD_RANGE); } // Update the conf iff the new value is greater than the existing val private void setS3ConfIfGreater(Configuration conf, String key, int newVal) { int maxVal = newVal; String curr = conf.get(key); if (curr != null && !curr.isEmpty()) { maxVal = Math.max(Integer.parseInt(curr), newVal); } LOG.info("File reader auto configured '{}={}'", key, maxVal); conf.set(key, Integer.toString(maxVal)); } // Update the conf iff the new value is greater than the existing val. This handles values that // may have suffixes (K, M, G, T, P, E) indicating well known bytes size suffixes private void setS3ConfIfGreater(Configuration conf, String key, long newVal) { long maxVal = conf.getLongBytes(key, newVal); maxVal = Math.max(maxVal, newVal); LOG.info("File reader auto configured '{}={}'", key, maxVal); conf.set(key, Long.toString(maxVal)); } } } ================================================ FILE: common/src/main/java/org/apache/comet/parquet/RowGroupFilter.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.parquet; import java.util.ArrayList; import java.util.List; import org.apache.parquet.filter2.compat.FilterCompat; import org.apache.parquet.filter2.compat.FilterCompat.Filter; import org.apache.parquet.filter2.compat.FilterCompat.NoOpFilter; import org.apache.parquet.filter2.compat.FilterCompat.Visitor; import org.apache.parquet.filter2.dictionarylevel.DictionaryFilter; import org.apache.parquet.filter2.predicate.FilterPredicate; import org.apache.parquet.filter2.predicate.SchemaCompatibilityValidator; import org.apache.parquet.filter2.statisticslevel.StatisticsFilter; import org.apache.parquet.hadoop.metadata.BlockMetaData; import org.apache.parquet.schema.MessageType; public class RowGroupFilter implements Visitor> { private final List blocks; private final MessageType schema; private final List levels; private final FileReader reader; public enum FilterLevel { STATISTICS, DICTIONARY, BLOOMFILTER } public static List filterRowGroups( List levels, Filter filter, List blocks, FileReader reader) { return filter.accept(new RowGroupFilter(levels, blocks, reader)); } public static List filterRowGroups( List levels, Filter filter, List blocks, MessageType schema) { return filter.accept(new RowGroupFilter(levels, blocks, schema)); } private RowGroupFilter(List levels, List blocks, FileReader reader) { this.levels = levels; this.blocks = blocks; this.reader = reader; this.schema = reader.getFileMetaData().getSchema(); } private RowGroupFilter(List levels, List blocks, MessageType schema) { this.levels = levels; this.blocks = blocks; this.reader = null; this.schema = schema; } @Override public List visit(FilterCompat.FilterPredicateCompat filterPredicateCompat) { FilterPredicate filterPredicate = filterPredicateCompat.getFilterPredicate(); // check that the schema of the filter matches the schema of the file SchemaCompatibilityValidator.validate(filterPredicate, schema); List filteredBlocks = new ArrayList<>(); for (BlockMetaData block : blocks) { boolean drop = false; if (levels.contains(FilterLevel.STATISTICS)) { drop = StatisticsFilter.canDrop(filterPredicate, block.getColumns()); } if (!drop && levels.contains(FilterLevel.DICTIONARY)) { drop = DictionaryFilter.canDrop( filterPredicate, block.getColumns(), new DictionaryPageReader( block, reader.getFileMetaData().getFileDecryptor(), reader.getInputStream(), reader.getOptions())); } if (!drop && levels.contains(FilterLevel.BLOOMFILTER)) { drop = filterPredicate.accept( new BloomFilterReader( block, reader.getFileMetaData().getFileDecryptor(), reader.getInputStream())); } if (!drop) { filteredBlocks.add(block); } } return filteredBlocks; } @Override public List visit( FilterCompat.UnboundRecordFilterCompat unboundRecordFilterCompat) { return blocks; } @Override public List visit(NoOpFilter noOpFilter) { return blocks; } } ================================================ FILE: common/src/main/java/org/apache/comet/parquet/RowGroupReader.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.parquet; import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.PrimitiveIterator; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.page.PageReadStore; import org.apache.parquet.column.page.PageReader; import org.apache.parquet.internal.filter2.columnindex.RowRanges; import org.apache.comet.IcebergApi; @IcebergApi public class RowGroupReader implements PageReadStore { private final Map readers = new HashMap<>(); private final long rowCount; private final RowRanges rowRanges; private final long rowIndexOffset; public RowGroupReader(long rowCount, long rowIndexOffset) { this.rowCount = rowCount; this.rowRanges = null; this.rowIndexOffset = rowIndexOffset; } RowGroupReader(RowRanges rowRanges) { this.rowRanges = rowRanges; this.rowCount = rowRanges.rowCount(); this.rowIndexOffset = -1; } @IcebergApi @Override public long getRowCount() { return rowCount; } @Override public PageReader getPageReader(ColumnDescriptor path) { return getPageReader(path.getPath()); } public PageReader getPageReader(String[] path) { final PageReader pageReader = readers.get(String.join(".", path)); if (pageReader == null) { throw new IllegalArgumentException( path + " is not found: " + readers.keySet() + " " + rowCount); } return pageReader; } @Override public Optional getRowIndexes() { return rowRanges == null ? Optional.empty() : Optional.of(rowRanges.iterator()); } @Override public Optional getRowIndexOffset() { return this.rowIndexOffset < 0L ? Optional.empty() : Optional.of(this.rowIndexOffset); } void addColumn(ColumnDescriptor path, ColumnPageReader reader) { if (readers.put(String.join(".", path.getPath()), reader) != null) { throw new IllegalStateException(path + " was already added"); } } } ================================================ FILE: common/src/main/java/org/apache/comet/parquet/TypeUtil.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.parquet; import java.util.Arrays; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.schema.*; import org.apache.parquet.schema.LogicalTypeAnnotation.*; import org.apache.spark.package$; import org.apache.spark.sql.execution.datasources.SchemaColumnConvertNotSupportedException; import org.apache.spark.sql.internal.SQLConf; import org.apache.spark.sql.types.*; import org.apache.comet.CometConf; import org.apache.comet.IcebergApi; import static org.apache.comet.parquet.Utils.descriptorToParquetColumnSpec; public class TypeUtil { /** * Converts the input Spark 'field' into a Parquet column descriptor. * * @see Comet Issue #2079 */ @IcebergApi public static ColumnDescriptor convertToParquet(StructField field) { Type.Repetition repetition; int maxDefinitionLevel; if (field.nullable()) { repetition = Type.Repetition.OPTIONAL; maxDefinitionLevel = 1; } else { repetition = Type.Repetition.REQUIRED; maxDefinitionLevel = 0; } String[] path = new String[] {field.name()}; DataType type = field.dataType(); Types.PrimitiveBuilder builder = null; // Only partition column can be `NullType`. Here we piggy-back onto Parquet boolean type // for constant vector of null values, we don't really care what Parquet type it is. if (type == DataTypes.BooleanType || type == DataTypes.NullType) { builder = Types.primitive(PrimitiveType.PrimitiveTypeName.BOOLEAN, repetition); } else if (type == DataTypes.IntegerType || type instanceof YearMonthIntervalType) { builder = Types.primitive(PrimitiveType.PrimitiveTypeName.INT32, repetition) .as(LogicalTypeAnnotation.intType(32, true)); } else if (type == DataTypes.DateType) { builder = Types.primitive(PrimitiveType.PrimitiveTypeName.INT32, repetition) .as(LogicalTypeAnnotation.dateType()); } else if (type == DataTypes.ByteType) { builder = Types.primitive(PrimitiveType.PrimitiveTypeName.INT32, repetition) .as(LogicalTypeAnnotation.intType(8, true)); } else if (type == DataTypes.ShortType) { builder = Types.primitive(PrimitiveType.PrimitiveTypeName.INT32, repetition) .as(LogicalTypeAnnotation.intType(16, true)); } else if (type == DataTypes.LongType) { builder = Types.primitive(PrimitiveType.PrimitiveTypeName.INT64, repetition); } else if (type == DataTypes.BinaryType) { builder = Types.primitive(PrimitiveType.PrimitiveTypeName.BINARY, repetition); } else if (type == DataTypes.StringType || (type.sameType(DataTypes.StringType) && isSpark40Plus())) { builder = Types.primitive(PrimitiveType.PrimitiveTypeName.BINARY, repetition) .as(LogicalTypeAnnotation.stringType()); } else if (type == DataTypes.FloatType) { builder = Types.primitive(PrimitiveType.PrimitiveTypeName.FLOAT, repetition); } else if (type == DataTypes.DoubleType) { builder = Types.primitive(PrimitiveType.PrimitiveTypeName.DOUBLE, repetition); } else if (type == DataTypes.TimestampType) { builder = Types.primitive(PrimitiveType.PrimitiveTypeName.INT64, repetition) .as(LogicalTypeAnnotation.timestampType(true, TimeUnit.MICROS)); } else if (type == TimestampNTZType$.MODULE$) { builder = Types.primitive(PrimitiveType.PrimitiveTypeName.INT64, repetition) .as(LogicalTypeAnnotation.timestampType(false, TimeUnit.MICROS)); } else if (type instanceof DecimalType) { DecimalType decimalType = (DecimalType) type; builder = Types.primitive(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY, repetition) .length(16) // always store as Decimal128 .as(LogicalTypeAnnotation.decimalType(decimalType.scale(), decimalType.precision())); } if (builder == null) { throw new UnsupportedOperationException("Unsupported input Spark type: " + type); } return new ColumnDescriptor(path, builder.named(field.name()), 0, maxDefinitionLevel); } public static ParquetColumnSpec convertToParquetSpec(StructField field) { return descriptorToParquetColumnSpec(convertToParquet(field)); } /** * Check whether the Parquet 'descriptor' and Spark read type 'sparkType' are compatible. If not, * throw exception. * *

This mostly follows the logic in Spark's * ParquetVectorUpdaterFactory#getUpdater(ColumnDescriptor, DataType) * * @param descriptor descriptor for a Parquet primitive column * @param sparkType Spark read type */ public static void checkParquetType(ColumnDescriptor descriptor, DataType sparkType) { PrimitiveType.PrimitiveTypeName typeName = descriptor.getPrimitiveType().getPrimitiveTypeName(); LogicalTypeAnnotation logicalTypeAnnotation = descriptor.getPrimitiveType().getLogicalTypeAnnotation(); boolean allowTypePromotion = (boolean) CometConf.COMET_SCHEMA_EVOLUTION_ENABLED().get(); if (sparkType instanceof NullType) { return; } switch (typeName) { case BOOLEAN: if (sparkType == DataTypes.BooleanType) return; break; case INT32: if (sparkType == DataTypes.IntegerType || canReadAsIntDecimal(descriptor, sparkType)) { return; } else if (sparkType == DataTypes.LongType && isUnsignedIntTypeMatched(logicalTypeAnnotation, 32)) { // In `ParquetToSparkSchemaConverter`, we map parquet UINT32 to our LongType. // For unsigned int32, it stores as plain signed int32 in Parquet when dictionary // fallbacks. We read them as long values. return; } else if (sparkType == DataTypes.LongType && allowTypePromotion) { // In Comet we allow schema evolution from int to long, if // `spark.comet.schemaEvolution.enabled` is enabled. return; } else if (sparkType == DataTypes.ByteType || sparkType == DataTypes.ShortType) { return; } else if (sparkType == DataTypes.DateType) { // TODO: use dateTimeRebaseMode from Spark side return; } else if (sparkType instanceof YearMonthIntervalType) { return; } else if (sparkType == DataTypes.DoubleType && isSpark40Plus()) { return; } else if (sparkType == TimestampNTZType$.MODULE$ && isSpark40Plus() && logicalTypeAnnotation instanceof DateLogicalTypeAnnotation) { return; } break; case INT64: if (sparkType == DataTypes.LongType || canReadAsLongDecimal(descriptor, sparkType)) { return; } else if (isLongDecimal(sparkType) && isUnsignedIntTypeMatched(logicalTypeAnnotation, 64)) { // In `ParquetToSparkSchemaConverter`, we map parquet UINT64 to our Decimal(20, 0). // For unsigned int64, it stores as plain signed int64 in Parquet when dictionary // fallbacks. We read them as decimal values. return; } else if (isTimestampTypeMatched(logicalTypeAnnotation, TimeUnit.MICROS) && (sparkType == TimestampNTZType$.MODULE$ || sparkType == DataTypes.TimestampType)) { validateTimestampType(logicalTypeAnnotation, sparkType); // TODO: use dateTimeRebaseMode from Spark side return; } else if (isTimestampTypeMatched(logicalTypeAnnotation, TimeUnit.MILLIS) && (sparkType == TimestampNTZType$.MODULE$ || sparkType == DataTypes.TimestampType)) { validateTimestampType(logicalTypeAnnotation, sparkType); return; } break; case INT96: if (sparkType == TimestampNTZType$.MODULE$) { if (isSpark40Plus()) return; // Spark 4.0+ supports Timestamp NTZ with INT96 convertErrorForTimestampNTZ(typeName.name()); } else if (sparkType == DataTypes.TimestampType) { return; } break; case FLOAT: if (sparkType == DataTypes.FloatType) return; // In Comet we allow schema evolution from float to double, if // `spark.comet.schemaEvolution.enabled` is enabled. if (sparkType == DataTypes.DoubleType && allowTypePromotion) return; break; case DOUBLE: if (sparkType == DataTypes.DoubleType) return; break; case BINARY: if (sparkType == DataTypes.StringType || sparkType == DataTypes.BinaryType || canReadAsBinaryDecimal(descriptor, sparkType)) { return; } if (sparkType.sameType(DataTypes.StringType) && isSpark40Plus()) { LogicalTypeAnnotation lta = descriptor.getPrimitiveType().getLogicalTypeAnnotation(); if (lta instanceof LogicalTypeAnnotation.StringLogicalTypeAnnotation) { return; } } break; case FIXED_LEN_BYTE_ARRAY: if (canReadAsIntDecimal(descriptor, sparkType) || canReadAsLongDecimal(descriptor, sparkType) || canReadAsBinaryDecimal(descriptor, sparkType) || sparkType == DataTypes.BinaryType // for uuid, since iceberg maps uuid to StringType || sparkType == DataTypes.StringType && logicalTypeAnnotation instanceof LogicalTypeAnnotation.UUIDLogicalTypeAnnotation) { return; } break; default: break; } throw new SchemaColumnConvertNotSupportedException( Arrays.toString(descriptor.getPath()), descriptor.getPrimitiveType().getPrimitiveTypeName().toString(), sparkType.catalogString()); } private static void validateTimestampType( LogicalTypeAnnotation logicalTypeAnnotation, DataType sparkType) { assert (logicalTypeAnnotation instanceof TimestampLogicalTypeAnnotation); // Throw an exception if the Parquet type is TimestampLTZ and the Catalyst type is TimestampNTZ. // This is to avoid mistakes in reading the timestamp values. if (((TimestampLogicalTypeAnnotation) logicalTypeAnnotation).isAdjustedToUTC() && sparkType == TimestampNTZType$.MODULE$ && !isSpark40Plus()) { convertErrorForTimestampNTZ("int64 time(" + logicalTypeAnnotation + ")"); } } private static void convertErrorForTimestampNTZ(String parquetType) { throw new RuntimeException( "Unable to create Parquet converter for data type " + TimestampNTZType$.MODULE$.json() + " whose Parquet type is " + parquetType); } private static boolean canReadAsIntDecimal(ColumnDescriptor descriptor, DataType dt) { if (!DecimalType.is32BitDecimalType(dt) && !(isSpark40Plus() && dt instanceof DecimalType)) return false; return isDecimalTypeMatched(descriptor, dt); } private static boolean canReadAsLongDecimal(ColumnDescriptor descriptor, DataType dt) { if (!DecimalType.is64BitDecimalType(dt) && !(isSpark40Plus() && dt instanceof DecimalType)) return false; return isDecimalTypeMatched(descriptor, dt); } private static boolean canReadAsBinaryDecimal(ColumnDescriptor descriptor, DataType dt) { if (!DecimalType.isByteArrayDecimalType(dt)) return false; return isDecimalTypeMatched(descriptor, dt); } private static boolean isLongDecimal(DataType dt) { if (dt instanceof DecimalType) { DecimalType d = (DecimalType) dt; return d.precision() == 20 && d.scale() == 0; } return false; } private static boolean isDecimalTypeMatched(ColumnDescriptor descriptor, DataType dt) { DecimalType d = (DecimalType) dt; LogicalTypeAnnotation typeAnnotation = descriptor.getPrimitiveType().getLogicalTypeAnnotation(); if (typeAnnotation instanceof DecimalLogicalTypeAnnotation) { DecimalLogicalTypeAnnotation decimalType = (DecimalLogicalTypeAnnotation) typeAnnotation; // It's OK if the required decimal precision is larger than or equal to the physical decimal // precision in the Parquet metadata, as long as the decimal scale is the same. return (decimalType.getPrecision() <= d.precision() && decimalType.getScale() == d.scale()) || (isSpark40Plus() && (!SQLConf.get().parquetVectorizedReaderEnabled() || (decimalType.getScale() <= d.scale() && decimalType.getPrecision() - decimalType.getScale() <= d.precision() - d.scale()))); } else if (isSpark40Plus()) { boolean isNullTypeAnnotation = typeAnnotation == null; boolean isIntTypeAnnotation = typeAnnotation instanceof IntLogicalTypeAnnotation; if (!SQLConf.get().parquetVectorizedReaderEnabled()) { return isNullTypeAnnotation || isIntTypeAnnotation; } else if (isNullTypeAnnotation || (isIntTypeAnnotation && ((IntLogicalTypeAnnotation) typeAnnotation).isSigned())) { PrimitiveType.PrimitiveTypeName typeName = descriptor.getPrimitiveType().getPrimitiveTypeName(); int integerPrecision = d.precision() - d.scale(); switch (typeName) { case INT32: return integerPrecision >= DecimalType$.MODULE$.IntDecimal().precision(); case INT64: return integerPrecision >= DecimalType$.MODULE$.LongDecimal().precision(); } } } return false; } private static boolean isTimestampTypeMatched( LogicalTypeAnnotation logicalTypeAnnotation, LogicalTypeAnnotation.TimeUnit unit) { return logicalTypeAnnotation instanceof TimestampLogicalTypeAnnotation && ((TimestampLogicalTypeAnnotation) logicalTypeAnnotation).getUnit() == unit; } private static boolean isUnsignedIntTypeMatched( LogicalTypeAnnotation logicalTypeAnnotation, int bitWidth) { return logicalTypeAnnotation instanceof IntLogicalTypeAnnotation && !((IntLogicalTypeAnnotation) logicalTypeAnnotation).isSigned() && ((IntLogicalTypeAnnotation) logicalTypeAnnotation).getBitWidth() == bitWidth; } static boolean isSpark40Plus() { return package$.MODULE$.SPARK_VERSION().compareTo("4.0") >= 0; } } ================================================ FILE: common/src/main/java/org/apache/comet/parquet/Utils.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.parquet; import java.util.HashMap; import java.util.Map; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.PrimitiveType; import org.apache.parquet.schema.Type; import org.apache.parquet.schema.Types; import org.apache.spark.sql.types.*; import org.apache.comet.CometSchemaImporter; import org.apache.comet.IcebergApi; public class Utils { /** This method is called from Apache Iceberg. */ @IcebergApi public static ColumnReader getColumnReader( DataType type, ParquetColumnSpec columnSpec, CometSchemaImporter importer, int batchSize, boolean useDecimal128, boolean useLazyMaterialization, boolean useLegacyTimestamp) { ColumnDescriptor descriptor = buildColumnDescriptor(columnSpec); return getColumnReader( type, descriptor, importer, batchSize, useDecimal128, useLazyMaterialization, useLegacyTimestamp); } /** * This method is called from Apache Iceberg. * * @see Comet Issue #2079 */ @IcebergApi public static ColumnReader getColumnReader( DataType type, ColumnDescriptor descriptor, CometSchemaImporter importer, int batchSize, boolean useDecimal128, boolean useLazyMaterialization) { // TODO: support `useLegacyDateTimestamp` for Iceberg return getColumnReader( type, descriptor, importer, batchSize, useDecimal128, useLazyMaterialization, true); } public static ColumnReader getColumnReader( DataType type, ColumnDescriptor descriptor, CometSchemaImporter importer, int batchSize, boolean useDecimal128, boolean useLazyMaterialization, boolean useLegacyDateTimestamp) { if (useLazyMaterialization && supportLazyMaterialization(type)) { return new LazyColumnReader( type, descriptor, importer, batchSize, useDecimal128, useLegacyDateTimestamp); } else { return new ColumnReader( type, descriptor, importer, batchSize, useDecimal128, useLegacyDateTimestamp); } } private static boolean supportLazyMaterialization(DataType type) { return (type instanceof StringType || type instanceof BinaryType); } /** * Initialize the Comet native Parquet reader. * * @param descriptor the Parquet column descriptor for the column to be read * @param readType the Spark read type used for type promotion. Null if promotion is not enabled. * @param batchSize the batch size, i.e., maximum number of elements per record batch * @param useDecimal128 whether to always represent decimals using 128 bits. If false, the native * reader may represent decimals using 32 or 64 bits, depending on the precision. * @param useLegacyDateTimestampOrNTZ whether to read dates/timestamps that were written in the * legacy hybrid Julian + Gregorian calendar as it is. If false, throw exceptions instead. If * the spark type is TimestampNTZ, this should be true. */ public static long initColumnReader( ColumnDescriptor descriptor, DataType readType, int batchSize, boolean useDecimal128, boolean useLegacyDateTimestampOrNTZ) { PrimitiveType primitiveType = descriptor.getPrimitiveType(); int primitiveTypeId = getPhysicalTypeId(primitiveType.getPrimitiveTypeName()); LogicalTypeAnnotation annotation = primitiveType.getLogicalTypeAnnotation(); // Process logical type information int bitWidth = -1; boolean isSigned = false; if (annotation instanceof LogicalTypeAnnotation.IntLogicalTypeAnnotation) { LogicalTypeAnnotation.IntLogicalTypeAnnotation intAnnotation = (LogicalTypeAnnotation.IntLogicalTypeAnnotation) annotation; bitWidth = intAnnotation.getBitWidth(); isSigned = intAnnotation.isSigned(); } int precision, scale; precision = scale = -1; if (annotation instanceof LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) { LogicalTypeAnnotation.DecimalLogicalTypeAnnotation decimalAnnotation = (LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) annotation; precision = decimalAnnotation.getPrecision(); scale = decimalAnnotation.getScale(); } int tu = -1; boolean isAdjustedUtc = false; if (annotation instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) { LogicalTypeAnnotation.TimestampLogicalTypeAnnotation timestampAnnotation = (LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) annotation; tu = getTimeUnitId(timestampAnnotation.getUnit()); isAdjustedUtc = timestampAnnotation.isAdjustedToUTC(); } TypePromotionInfo promotionInfo; if (readType != null) { promotionInfo = new TypePromotionInfo(readType); } else { // If type promotion is not enable, we'll just use the Parquet primitive type and precision. promotionInfo = new TypePromotionInfo(primitiveTypeId, precision, scale, bitWidth); } return Native.initColumnReader( primitiveTypeId, getLogicalTypeId(annotation), promotionInfo.physicalTypeId, descriptor.getPath(), descriptor.getMaxDefinitionLevel(), descriptor.getMaxRepetitionLevel(), bitWidth, promotionInfo.bitWidth, isSigned, primitiveType.getTypeLength(), precision, promotionInfo.precision, scale, promotionInfo.scale, tu, isAdjustedUtc, batchSize, useDecimal128, useLegacyDateTimestampOrNTZ); } static class TypePromotionInfo { // The Parquet physical type ID converted from the Spark read schema, or the original Parquet // physical type ID if type promotion is not enabled. int physicalTypeId; // Decimal precision from the Spark read schema, or -1 if it's not decimal type. int precision; // Decimal scale from the Spark read schema, or -1 if it's not decimal type. int scale; // Integer bit width from the Spark read schema, or -1 if it's not integer type. int bitWidth; TypePromotionInfo(int physicalTypeId, int precision, int scale, int bitWidth) { this.physicalTypeId = physicalTypeId; this.precision = precision; this.scale = scale; this.bitWidth = bitWidth; } TypePromotionInfo(DataType sparkReadType) { // Create a dummy `StructField` from the input Spark type. We don't care about // field name, nullability and metadata. StructField f = new StructField("f", sparkReadType, false, Metadata.empty()); ColumnDescriptor descriptor = TypeUtil.convertToParquet(f); PrimitiveType primitiveType = descriptor.getPrimitiveType(); int physicalTypeId = getPhysicalTypeId(primitiveType.getPrimitiveTypeName()); LogicalTypeAnnotation annotation = primitiveType.getLogicalTypeAnnotation(); int precision = -1; int scale = -1; int bitWidth = -1; if (annotation instanceof LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) { LogicalTypeAnnotation.DecimalLogicalTypeAnnotation decimalAnnotation = (LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) annotation; precision = decimalAnnotation.getPrecision(); scale = decimalAnnotation.getScale(); } if (annotation instanceof LogicalTypeAnnotation.IntLogicalTypeAnnotation) { LogicalTypeAnnotation.IntLogicalTypeAnnotation intAnnotation = (LogicalTypeAnnotation.IntLogicalTypeAnnotation) annotation; bitWidth = intAnnotation.getBitWidth(); } this.physicalTypeId = physicalTypeId; this.precision = precision; this.scale = scale; this.bitWidth = bitWidth; } } /** * Maps the input Parquet physical type 'typeName' to an integer representing it. This is used for * serialization between the Java and native side. * * @param typeName enum for the Parquet physical type * @return an integer representing the input physical type */ static int getPhysicalTypeId(PrimitiveType.PrimitiveTypeName typeName) { switch (typeName) { case BOOLEAN: return 0; case INT32: return 1; case INT64: return 2; case INT96: return 3; case FLOAT: return 4; case DOUBLE: return 5; case BINARY: return 6; case FIXED_LEN_BYTE_ARRAY: return 7; } throw new IllegalArgumentException("Invalid Parquet physical type: " + typeName); } /** * Maps the input Parquet logical type 'annotation' to an integer representing it. This is used * for serialization between the Java and native side. * * @param annotation the Parquet logical type annotation * @return an integer representing the input logical type */ static int getLogicalTypeId(LogicalTypeAnnotation annotation) { if (annotation == null) { return -1; // No logical type associated } else if (annotation instanceof LogicalTypeAnnotation.IntLogicalTypeAnnotation) { return 0; } else if (annotation instanceof LogicalTypeAnnotation.StringLogicalTypeAnnotation) { return 1; } else if (annotation instanceof LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) { return 2; } else if (annotation instanceof LogicalTypeAnnotation.DateLogicalTypeAnnotation) { return 3; } else if (annotation instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) { return 4; } else if (annotation instanceof LogicalTypeAnnotation.EnumLogicalTypeAnnotation) { return 5; } else if (annotation instanceof LogicalTypeAnnotation.UUIDLogicalTypeAnnotation) { return 6; } throw new UnsupportedOperationException("Unsupported Parquet logical type " + annotation); } static int getTimeUnitId(LogicalTypeAnnotation.TimeUnit tu) { switch (tu) { case MILLIS: return 0; case MICROS: return 1; case NANOS: return 2; default: throw new UnsupportedOperationException("Unsupported TimeUnit " + tu); } } @IcebergApi public static ColumnDescriptor buildColumnDescriptor(ParquetColumnSpec columnSpec) { PrimitiveType.PrimitiveTypeName primType = PrimitiveType.PrimitiveTypeName.valueOf(columnSpec.getPhysicalType()); Type.Repetition repetition; if (columnSpec.getMaxRepetitionLevel() > 0) { repetition = Type.Repetition.REPEATED; } else if (columnSpec.getMaxDefinitionLevel() > 0) { repetition = Type.Repetition.OPTIONAL; } else { repetition = Type.Repetition.REQUIRED; } String name = columnSpec.getPath()[columnSpec.getPath().length - 1]; // Reconstruct the logical type from parameters LogicalTypeAnnotation logicalType = null; if (columnSpec.getLogicalTypeName() != null) { logicalType = reconstructLogicalType( columnSpec.getLogicalTypeName(), columnSpec.getLogicalTypeParams()); } PrimitiveType primitiveType; if (primType == PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) { primitiveType = Types.primitive(primType, repetition) .length(columnSpec.getTypeLength()) .as(logicalType) .id(columnSpec.getFieldId()) .named(name); } else { primitiveType = Types.primitive(primType, repetition) .as(logicalType) .id(columnSpec.getFieldId()) .named(name); } return new ColumnDescriptor( columnSpec.getPath(), primitiveType, columnSpec.getMaxRepetitionLevel(), columnSpec.getMaxDefinitionLevel()); } private static LogicalTypeAnnotation reconstructLogicalType( String logicalTypeName, java.util.Map params) { switch (logicalTypeName) { // MAP case "MapLogicalTypeAnnotation": return LogicalTypeAnnotation.mapType(); // LIST case "ListLogicalTypeAnnotation": return LogicalTypeAnnotation.listType(); // STRING case "StringLogicalTypeAnnotation": return LogicalTypeAnnotation.stringType(); // MAP_KEY_VALUE case "MapKeyValueLogicalTypeAnnotation": return LogicalTypeAnnotation.MapKeyValueTypeAnnotation.getInstance(); // ENUM case "EnumLogicalTypeAnnotation": return LogicalTypeAnnotation.enumType(); // DECIMAL case "DecimalLogicalTypeAnnotation": if (!params.containsKey("scale") || !params.containsKey("precision")) { throw new IllegalArgumentException( "Missing required parameters for DecimalLogicalTypeAnnotation: " + params); } int scale = Integer.parseInt(params.get("scale")); int precision = Integer.parseInt(params.get("precision")); return LogicalTypeAnnotation.decimalType(scale, precision); // DATE case "DateLogicalTypeAnnotation": return LogicalTypeAnnotation.dateType(); // TIME case "TimeLogicalTypeAnnotation": if (!params.containsKey("isAdjustedToUTC") || !params.containsKey("unit")) { throw new IllegalArgumentException( "Missing required parameters for TimeLogicalTypeAnnotation: " + params); } boolean isUTC = Boolean.parseBoolean(params.get("isAdjustedToUTC")); String timeUnitStr = params.get("unit"); LogicalTypeAnnotation.TimeUnit timeUnit; switch (timeUnitStr) { case "MILLIS": timeUnit = LogicalTypeAnnotation.TimeUnit.MILLIS; break; case "MICROS": timeUnit = LogicalTypeAnnotation.TimeUnit.MICROS; break; case "NANOS": timeUnit = LogicalTypeAnnotation.TimeUnit.NANOS; break; default: throw new IllegalArgumentException("Unknown time unit: " + timeUnitStr); } return LogicalTypeAnnotation.timeType(isUTC, timeUnit); // TIMESTAMP case "TimestampLogicalTypeAnnotation": if (!params.containsKey("isAdjustedToUTC") || !params.containsKey("unit")) { throw new IllegalArgumentException( "Missing required parameters for TimestampLogicalTypeAnnotation: " + params); } boolean isAdjustedToUTC = Boolean.parseBoolean(params.get("isAdjustedToUTC")); String unitStr = params.get("unit"); LogicalTypeAnnotation.TimeUnit unit; switch (unitStr) { case "MILLIS": unit = LogicalTypeAnnotation.TimeUnit.MILLIS; break; case "MICROS": unit = LogicalTypeAnnotation.TimeUnit.MICROS; break; case "NANOS": unit = LogicalTypeAnnotation.TimeUnit.NANOS; break; default: throw new IllegalArgumentException("Unknown timestamp unit: " + unitStr); } return LogicalTypeAnnotation.timestampType(isAdjustedToUTC, unit); // INTEGER case "IntLogicalTypeAnnotation": if (!params.containsKey("isSigned") || !params.containsKey("bitWidth")) { throw new IllegalArgumentException( "Missing required parameters for IntLogicalTypeAnnotation: " + params); } boolean isSigned = Boolean.parseBoolean(params.get("isSigned")); int bitWidth = Integer.parseInt(params.get("bitWidth")); return LogicalTypeAnnotation.intType(bitWidth, isSigned); // JSON case "JsonLogicalTypeAnnotation": return LogicalTypeAnnotation.jsonType(); // BSON case "BsonLogicalTypeAnnotation": return LogicalTypeAnnotation.bsonType(); // UUID case "UUIDLogicalTypeAnnotation": return LogicalTypeAnnotation.uuidType(); // INTERVAL case "IntervalLogicalTypeAnnotation": return LogicalTypeAnnotation.IntervalLogicalTypeAnnotation.getInstance(); default: throw new IllegalArgumentException("Unknown logical type: " + logicalTypeName); } } @IcebergApi public static ParquetColumnSpec descriptorToParquetColumnSpec(ColumnDescriptor descriptor) { String[] path = descriptor.getPath(); PrimitiveType primitiveType = descriptor.getPrimitiveType(); String physicalType = primitiveType.getPrimitiveTypeName().name(); int typeLength = primitiveType.getPrimitiveTypeName() == PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY ? primitiveType.getTypeLength() : 0; boolean isRepeated = primitiveType.getRepetition() == Type.Repetition.REPEATED; String logicalTypeName = null; Map logicalTypeParams = new HashMap<>(); LogicalTypeAnnotation logicalType = primitiveType.getLogicalTypeAnnotation(); if (logicalType != null) { logicalTypeName = logicalType.getClass().getSimpleName(); // Handle specific logical types if (logicalType instanceof LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) { LogicalTypeAnnotation.DecimalLogicalTypeAnnotation decimal = (LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) logicalType; logicalTypeParams.put("precision", String.valueOf(decimal.getPrecision())); logicalTypeParams.put("scale", String.valueOf(decimal.getScale())); } else if (logicalType instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) { LogicalTypeAnnotation.TimestampLogicalTypeAnnotation timestamp = (LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) logicalType; logicalTypeParams.put("isAdjustedToUTC", String.valueOf(timestamp.isAdjustedToUTC())); logicalTypeParams.put("unit", timestamp.getUnit().name()); } else if (logicalType instanceof LogicalTypeAnnotation.TimeLogicalTypeAnnotation) { LogicalTypeAnnotation.TimeLogicalTypeAnnotation time = (LogicalTypeAnnotation.TimeLogicalTypeAnnotation) logicalType; logicalTypeParams.put("isAdjustedToUTC", String.valueOf(time.isAdjustedToUTC())); logicalTypeParams.put("unit", time.getUnit().name()); } else if (logicalType instanceof LogicalTypeAnnotation.IntLogicalTypeAnnotation) { LogicalTypeAnnotation.IntLogicalTypeAnnotation intType = (LogicalTypeAnnotation.IntLogicalTypeAnnotation) logicalType; logicalTypeParams.put("isSigned", String.valueOf(intType.isSigned())); logicalTypeParams.put("bitWidth", String.valueOf(intType.getBitWidth())); } } int id = -1; Type type = descriptor.getPrimitiveType(); if (type != null && type.getId() != null) { id = type.getId().intValue(); } return new ParquetColumnSpec( id, path, physicalType, typeLength, isRepeated, descriptor.getMaxDefinitionLevel(), descriptor.getMaxRepetitionLevel(), logicalTypeName, logicalTypeParams); } } ================================================ FILE: common/src/main/java/org/apache/comet/parquet/WrappedInputFile.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.parquet; import java.io.IOException; import java.io.InputStream; import java.lang.reflect.Method; import org.apache.parquet.io.InputFile; import org.apache.parquet.io.SeekableInputStream; import org.apache.comet.IcebergApi; /** * Wraps an Object that possibly implements the methods of a Parquet InputFile (but is not a Parquet * InputFile). Such an object` exists, for instance, in Iceberg's InputFile */ @IcebergApi public class WrappedInputFile implements InputFile { Object wrapped; @IcebergApi public WrappedInputFile(Object inputFile) { this.wrapped = inputFile; } @Override public long getLength() throws IOException { try { Method targetMethod = wrapped.getClass().getDeclaredMethod("getLength"); // targetMethod.setAccessible(true); return (long) targetMethod.invoke(wrapped); } catch (Exception e) { throw new IOException(e); } } @Override public SeekableInputStream newStream() throws IOException { try { Method targetMethod = wrapped.getClass().getDeclaredMethod("newStream"); // targetMethod.setAccessible(true); InputStream stream = (InputStream) targetMethod.invoke(wrapped); return new WrappedSeekableInputStream(stream); } catch (Exception e) { throw new IOException(e); } } @Override public String toString() { return wrapped.toString(); } } ================================================ FILE: common/src/main/java/org/apache/comet/parquet/WrappedSeekableInputStream.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.parquet; import java.io.IOException; import java.io.InputStream; import java.lang.reflect.Method; import java.util.Objects; import org.apache.parquet.io.DelegatingSeekableInputStream; /** * Wraps an InputStream that possibly implements the methods of a Parquet SeekableInputStream (but * is not a Parquet SeekableInputStream). Such an InputStream exists, for instance, in Iceberg's * SeekableInputStream */ public class WrappedSeekableInputStream extends DelegatingSeekableInputStream { private final InputStream wrappedInputStream; // The InputStream we are wrapping public WrappedSeekableInputStream(InputStream inputStream) { super(inputStream); this.wrappedInputStream = Objects.requireNonNull(inputStream, "InputStream cannot be null"); } @Override public long getPos() throws IOException { try { Method targetMethod = wrappedInputStream.getClass().getDeclaredMethod("getPos"); // targetMethod.setAccessible(true); return (long) targetMethod.invoke(wrappedInputStream); } catch (Exception e) { throw new IOException(e); } } @Override public void seek(long newPos) throws IOException { try { Method targetMethod = wrappedInputStream.getClass().getDeclaredMethod("seek", long.class); targetMethod.setAccessible(true); targetMethod.invoke(wrappedInputStream, newPos); } catch (Exception e) { throw new IOException(e); } } } ================================================ FILE: common/src/main/java/org/apache/comet/vector/CometDecodedVector.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.vector; import org.apache.arrow.vector.BaseVariableWidthVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.types.pojo.Field; import org.apache.spark.sql.comet.util.Utils; import org.apache.spark.unsafe.Platform; /** A Comet vector whose elements are already decoded (i.e., materialized). */ public abstract class CometDecodedVector extends CometVector { /** * The vector that stores all the values. For dictionary-backed vector, this is the vector of * indices. */ protected final ValueVector valueVector; private boolean hasNull; private int numNulls; private int numValues; private int validityByteCacheIndex = -1; private byte validityByteCache; protected boolean isUuid; protected CometDecodedVector(ValueVector vector, Field valueField, boolean useDecimal128) { this(vector, valueField, useDecimal128, false); } protected CometDecodedVector( ValueVector vector, Field valueField, boolean useDecimal128, boolean isUuid) { super(Utils.fromArrowField(valueField), useDecimal128); this.valueVector = vector; this.numNulls = valueVector.getNullCount(); this.numValues = valueVector.getValueCount(); this.hasNull = numNulls != 0; this.isUuid = isUuid; } @Override public ValueVector getValueVector() { return valueVector; } @Override public void setNumNulls(int numNulls) { // We don't need to update null count in 'valueVector' since 'ValueVector.getNullCount' will // re-compute the null count from validity buffer. this.numNulls = numNulls; this.hasNull = numNulls != 0; this.validityByteCacheIndex = -1; } @Override public void setNumValues(int numValues) { this.numValues = numValues; if (valueVector instanceof BaseVariableWidthVector) { BaseVariableWidthVector bv = (BaseVariableWidthVector) valueVector; // In case `lastSet` is smaller than `numValues`, `setValueCount` will set all the offsets // within `[lastSet + 1, numValues)` to be empty, which is incorrect in our case. // // For instance, this can happen if one first call `setNumValues` with input 100, and then // again `setNumValues` with 200. The first call will set `lastSet` to 99, while the second // call will set all strings between indices `[100, 200)` to be empty. bv.setLastSet(numValues); } valueVector.setValueCount(numValues); } public int numValues() { return numValues; } @Override public boolean hasNull() { return hasNull; } @Override public int numNulls() { return numNulls; } @Override public boolean isNullAt(int rowId) { if (!hasNull) return false; int byteIndex = rowId >> 3; if (byteIndex != validityByteCacheIndex) { long validityBufferAddress = valueVector.getValidityBuffer().memoryAddress(); validityByteCache = Platform.getByte(null, validityBufferAddress + byteIndex); validityByteCacheIndex = byteIndex; } return ((validityByteCache >> (rowId & 7)) & 1) == 0; } } ================================================ FILE: common/src/main/java/org/apache/comet/vector/CometDelegateVector.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.vector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.dictionary.DictionaryProvider; import org.apache.spark.sql.types.DataType; import org.apache.spark.sql.types.Decimal; import org.apache.spark.sql.vectorized.ColumnVector; import org.apache.spark.sql.vectorized.ColumnarArray; import org.apache.spark.sql.vectorized.ColumnarMap; import org.apache.spark.unsafe.types.UTF8String; /** A special Comet vector that just delegate all calls */ public class CometDelegateVector extends CometVector { protected CometVector delegate; public CometDelegateVector(DataType dataType) { this(dataType, null, false); } public CometDelegateVector(DataType dataType, boolean useDecimal128) { this(dataType, null, useDecimal128); } public CometDelegateVector(DataType dataType, CometVector delegate, boolean useDecimal128) { super(dataType, useDecimal128); if (delegate instanceof CometDelegateVector) { throw new IllegalArgumentException("cannot have nested delegation"); } this.delegate = delegate; } protected void setDelegate(CometVector delegate) { this.delegate = delegate; } @Override public void setNumNulls(int numNulls) { delegate.setNumNulls(numNulls); } @Override public void setNumValues(int numValues) { delegate.setNumValues(numValues); } @Override public int numValues() { return delegate.numValues(); } @Override public boolean hasNull() { return delegate.hasNull(); } @Override public int numNulls() { return delegate.numNulls(); } @Override public boolean isNullAt(int rowId) { return delegate.isNullAt(rowId); } @Override public boolean getBoolean(int rowId) { return delegate.getBoolean(rowId); } @Override public byte getByte(int rowId) { return delegate.getByte(rowId); } @Override public short getShort(int rowId) { return delegate.getShort(rowId); } @Override public int getInt(int rowId) { return delegate.getInt(rowId); } @Override public long getLong(int rowId) { return delegate.getLong(rowId); } @Override public long getLongDecimal(int rowId) { return delegate.getLongDecimal(rowId); } @Override public float getFloat(int rowId) { return delegate.getFloat(rowId); } @Override public double getDouble(int rowId) { return delegate.getDouble(rowId); } @Override public Decimal getDecimal(int i, int precision, int scale) { return delegate.getDecimal(i, precision, scale); } @Override byte[] getBinaryDecimal(int i) { return delegate.getBinaryDecimal(i); } @Override public UTF8String getUTF8String(int rowId) { return delegate.getUTF8String(rowId); } @Override public byte[] getBinary(int rowId) { return delegate.getBinary(rowId); } @Override public ColumnarArray getArray(int i) { return delegate.getArray(i); } @Override public ColumnarMap getMap(int i) { return delegate.getMap(i); } @Override public ColumnVector getChild(int i) { return delegate.getChild(i); } @Override public ValueVector getValueVector() { return delegate.getValueVector(); } @Override public CometVector slice(int offset, int length) { return delegate.slice(offset, length); } @Override public DictionaryProvider getDictionaryProvider() { return delegate.getDictionaryProvider(); } } ================================================ FILE: common/src/main/java/org/apache/comet/vector/CometDictionary.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.vector; import org.apache.arrow.vector.ValueVector; import org.apache.spark.unsafe.types.UTF8String; /** A dictionary which maps indices (integers) to values. */ public class CometDictionary implements AutoCloseable { private static final int DECIMAL_BYTE_WIDTH = 16; private CometPlainVector values; private final int numValues; /** Decoded dictionary values. We only need to copy values for decimal type. */ private volatile ByteArrayWrapper[] binaries; public CometDictionary(CometPlainVector values) { this.values = values; this.numValues = values.numValues(); } public void setDictionaryVector(CometPlainVector values) { this.values = values; if (values.numValues() != numValues) { throw new IllegalArgumentException("Mismatched dictionary size"); } } public ValueVector getValueVector() { return values.getValueVector(); } public boolean decodeToBoolean(int index) { return values.getBoolean(index); } public byte decodeToByte(int index) { return values.getByte(index); } public short decodeToShort(int index) { return values.getShort(index); } public int decodeToInt(int index) { return values.getInt(index); } public long decodeToLong(int index) { return values.getLong(index); } public long decodeToLongDecimal(int index) { return values.getLongDecimal(index); } public float decodeToFloat(int index) { return values.getFloat(index); } public double decodeToDouble(int index) { return values.getDouble(index); } public byte[] decodeToBinary(int index) { switch (values.getValueVector().getMinorType()) { case VARBINARY: case FIXEDSIZEBINARY: return values.getBinary(index); case DECIMAL: if (binaries == null) { // We only need to copy values for decimal 128 type as random access // to the dictionary is not efficient for decimal (it needs to copy // the value to a new byte array everytime). ByteArrayWrapper[] binaries = new ByteArrayWrapper[numValues]; for (int i = 0; i < numValues; i++) { // Need copying here since we re-use byte array for decimal byte[] bytes = new byte[DECIMAL_BYTE_WIDTH]; bytes = values.copyBinaryDecimal(i, bytes); binaries[i] = new ByteArrayWrapper(bytes); } this.binaries = binaries; } return binaries[index].bytes; default: throw new IllegalArgumentException( "Invalid Arrow minor type: " + values.getValueVector().getMinorType()); } } public UTF8String decodeToUTF8String(int index) { return values.getUTF8String(index); } @Override public void close() { values.close(); } private static class ByteArrayWrapper { private final byte[] bytes; ByteArrayWrapper(byte[] bytes) { this.bytes = bytes; } } } ================================================ FILE: common/src/main/java/org/apache/comet/vector/CometDictionaryVector.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.vector; import org.apache.arrow.vector.IntVector; import org.apache.arrow.vector.dictionary.DictionaryProvider; import org.apache.arrow.vector.util.TransferPair; import org.apache.parquet.Preconditions; import org.apache.spark.unsafe.types.UTF8String; /** A column vector whose elements are dictionary-encoded. */ public class CometDictionaryVector extends CometDecodedVector { public final CometPlainVector indices; public final CometDictionary values; public final DictionaryProvider provider; /** Whether this vector is an alias sliced from another vector. */ private final boolean isAlias; public CometDictionaryVector( CometPlainVector indices, CometDictionary values, DictionaryProvider provider, boolean useDecimal128) { this(indices, values, provider, useDecimal128, false, false); } public CometDictionaryVector( CometPlainVector indices, CometDictionary values, DictionaryProvider provider, boolean useDecimal128, boolean isAlias, boolean isUuid) { super(indices.valueVector, values.getValueVector().getField(), useDecimal128, isUuid); Preconditions.checkArgument( indices.valueVector instanceof IntVector, "'indices' should be a IntVector"); this.values = values; this.indices = indices; this.provider = provider; this.isAlias = isAlias; } @Override public DictionaryProvider getDictionaryProvider() { return this.provider; } @Override public void close() { super.close(); // Only close the values vector if this is not a sliced vector. if (!isAlias) { values.close(); } } @Override public boolean getBoolean(int i) { return values.decodeToBoolean(indices.getInt(i)); } @Override public byte getByte(int i) { return values.decodeToByte(indices.getInt(i)); } @Override public short getShort(int i) { return values.decodeToShort(indices.getInt(i)); } @Override public int getInt(int i) { return values.decodeToInt(indices.getInt(i)); } @Override public long getLong(int i) { return values.decodeToLong(indices.getInt(i)); } @Override public long getLongDecimal(int i) { return values.decodeToLongDecimal(indices.getInt(i)); } @Override public float getFloat(int i) { return values.decodeToFloat(indices.getInt(i)); } @Override public double getDouble(int i) { return values.decodeToDouble(indices.getInt(i)); } @Override public UTF8String getUTF8String(int i) { return values.decodeToUTF8String(indices.getInt(i)); } @Override public byte[] getBinary(int i) { return values.decodeToBinary(indices.getInt(i)); } @Override byte[] getBinaryDecimal(int i) { return values.decodeToBinary(indices.getInt(i)); } @Override public CometVector slice(int offset, int length) { TransferPair tp = indices.valueVector.getTransferPair(indices.valueVector.getAllocator()); tp.splitAndTransfer(offset, length); CometPlainVector sliced = new CometPlainVector(tp.getTo(), useDecimal128); // Set the alias flag to true so that the sliced vector will not close the dictionary vector. // Otherwise, if the dictionary is closed, the sliced vector will not be able to access the // dictionary. return new CometDictionaryVector(sliced, values, provider, useDecimal128, true, isUuid); } } ================================================ FILE: common/src/main/java/org/apache/comet/vector/CometLazyVector.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.vector; import org.apache.arrow.vector.ValueVector; import org.apache.spark.sql.types.DataType; import org.apache.comet.parquet.LazyColumnReader; public class CometLazyVector extends CometDelegateVector { private final LazyColumnReader columnReader; public CometLazyVector(DataType type, LazyColumnReader columnReader, boolean useDecimal128) { super(type, useDecimal128); this.columnReader = columnReader; } public CometDecodedVector getDecodedVector() { return (CometDecodedVector) delegate; } @Override public ValueVector getValueVector() { columnReader.readAllBatch(); setDelegate(columnReader.loadVector()); return super.getValueVector(); } @Override public void setNumNulls(int numNulls) { throw new UnsupportedOperationException("CometLazyVector doesn't support 'setNumNulls'"); } @Override public void setNumValues(int numValues) { throw new UnsupportedOperationException("CometLazyVector doesn't support 'setNumValues'"); } @Override public void close() { // Do nothing. 'vector' is closed by 'columnReader' which owns it. } @Override public boolean hasNull() { columnReader.readAllBatch(); setDelegate(columnReader.loadVector()); return super.hasNull(); } @Override public int numNulls() { columnReader.readAllBatch(); setDelegate(columnReader.loadVector()); return super.numNulls(); } @Override public boolean isNullAt(int rowId) { if (columnReader.materializeUpToIfNecessary(rowId)) { setDelegate(columnReader.loadVector()); } return super.isNullAt(rowId); } } ================================================ FILE: common/src/main/java/org/apache/comet/vector/CometListVector.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.vector; import org.apache.arrow.vector.*; import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.dictionary.DictionaryProvider; import org.apache.arrow.vector.util.TransferPair; import org.apache.spark.sql.vectorized.ColumnVector; import org.apache.spark.sql.vectorized.ColumnarArray; /** A Comet column vector for list type. */ public class CometListVector extends CometDecodedVector { final ListVector listVector; final ValueVector dataVector; final ColumnVector dataColumnVector; final DictionaryProvider dictionaryProvider; public CometListVector( ValueVector vector, boolean useDecimal128, DictionaryProvider dictionaryProvider) { super(vector, vector.getField(), useDecimal128); this.listVector = ((ListVector) vector); this.dataVector = listVector.getDataVector(); this.dictionaryProvider = dictionaryProvider; this.dataColumnVector = getVector(dataVector, useDecimal128, dictionaryProvider); } @Override public ColumnarArray getArray(int i) { if (isNullAt(i)) return null; int start = listVector.getOffsetBuffer().getInt(i * ListVector.OFFSET_WIDTH); int end = listVector.getOffsetBuffer().getInt((i + 1) * ListVector.OFFSET_WIDTH); return new ColumnarArray(dataColumnVector, start, end - start); } @Override public CometVector slice(int offset, int length) { TransferPair tp = this.valueVector.getTransferPair(this.valueVector.getAllocator()); tp.splitAndTransfer(offset, length); return new CometListVector(tp.getTo(), useDecimal128, dictionaryProvider); } } ================================================ FILE: common/src/main/java/org/apache/comet/vector/CometMapVector.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.vector; import org.apache.arrow.vector.*; import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.dictionary.DictionaryProvider; import org.apache.arrow.vector.util.TransferPair; import org.apache.spark.sql.vectorized.ColumnVector; import org.apache.spark.sql.vectorized.ColumnarMap; /** A Comet column vector for map type. */ public class CometMapVector extends CometDecodedVector { final MapVector mapVector; final ValueVector dataVector; final CometStructVector dataColumnVector; final DictionaryProvider dictionaryProvider; final ColumnVector keys; final ColumnVector values; public CometMapVector( ValueVector vector, boolean useDecimal128, DictionaryProvider dictionaryProvider) { super(vector, vector.getField(), useDecimal128); this.mapVector = ((MapVector) vector); this.dataVector = mapVector.getDataVector(); this.dictionaryProvider = dictionaryProvider; if (dataVector instanceof StructVector) { this.dataColumnVector = new CometStructVector(dataVector, useDecimal128, dictionaryProvider); if (dataColumnVector.children.size() != 2) { throw new RuntimeException( "MapVector's dataVector should have 2 children, but got: " + dataColumnVector.children.size()); } this.keys = dataColumnVector.getChild(0); this.values = dataColumnVector.getChild(1); } else { throw new RuntimeException( "MapVector's dataVector should be StructVector, but got: " + dataVector.getClass().getSimpleName()); } } @Override public ColumnarMap getMap(int i) { if (isNullAt(i)) return null; int start = mapVector.getOffsetBuffer().getInt(i * MapVector.OFFSET_WIDTH); int end = mapVector.getOffsetBuffer().getInt((i + 1) * MapVector.OFFSET_WIDTH); return new ColumnarMap(keys, values, start, end - start); } @Override public CometVector slice(int offset, int length) { TransferPair tp = this.valueVector.getTransferPair(this.valueVector.getAllocator()); tp.splitAndTransfer(offset, length); return new CometMapVector(tp.getTo(), useDecimal128, dictionaryProvider); } } ================================================ FILE: common/src/main/java/org/apache/comet/vector/CometPlainVector.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.vector; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.UUID; import org.apache.arrow.c.CDataDictionaryProvider; import org.apache.arrow.vector.*; import org.apache.arrow.vector.util.TransferPair; import org.apache.parquet.Preconditions; import org.apache.spark.unsafe.Platform; import org.apache.spark.unsafe.types.UTF8String; /** A column vector whose elements are plainly decoded. */ public class CometPlainVector extends CometDecodedVector { private final long valueBufferAddress; private final boolean isBaseFixedWidthVector; private byte booleanByteCache; private int booleanByteCacheIndex = -1; private boolean isReused; public CometPlainVector(ValueVector vector, boolean useDecimal128) { this(vector, useDecimal128, false); } public CometPlainVector(ValueVector vector, boolean useDecimal128, boolean isUuid) { this(vector, useDecimal128, isUuid, false); } public CometPlainVector( ValueVector vector, boolean useDecimal128, boolean isUuid, boolean isReused) { super(vector, vector.getField(), useDecimal128, isUuid); // NullType doesn't have data buffer. if (vector instanceof NullVector) { this.valueBufferAddress = -1; } else { this.valueBufferAddress = vector.getDataBuffer().memoryAddress(); } isBaseFixedWidthVector = valueVector instanceof BaseFixedWidthVector; this.isReused = isReused; } public boolean isReused() { return isReused; } public void setReused(boolean isReused) { this.isReused = isReused; } @Override public void setNumNulls(int numNulls) { super.setNumNulls(numNulls); this.booleanByteCacheIndex = -1; } @Override public boolean getBoolean(int rowId) { int byteIndex = rowId >> 3; if (byteIndex != booleanByteCacheIndex) { booleanByteCache = getByte(byteIndex); booleanByteCacheIndex = byteIndex; } return ((booleanByteCache >> (rowId & 7)) & 1) == 1; } @Override public byte getByte(int rowId) { return Platform.getByte(null, valueBufferAddress + rowId); } @Override public short getShort(int rowId) { return Platform.getShort(null, valueBufferAddress + rowId * 2L); } @Override public int getInt(int rowId) { return Platform.getInt(null, valueBufferAddress + rowId * 4L); } @Override public long getLong(int rowId) { return Platform.getLong(null, valueBufferAddress + rowId * 8L); } @Override public long getLongDecimal(int rowId) { return Platform.getLong(null, valueBufferAddress + rowId * 16L); } @Override public float getFloat(int rowId) { return Platform.getFloat(null, valueBufferAddress + rowId * 4L); } @Override public double getDouble(int rowId) { return Platform.getDouble(null, valueBufferAddress + rowId * 8L); } @Override public UTF8String getUTF8String(int rowId) { if (isNullAt(rowId)) return null; if (!isBaseFixedWidthVector) { BaseVariableWidthVector varWidthVector = (BaseVariableWidthVector) valueVector; long offsetBufferAddress = varWidthVector.getOffsetBuffer().memoryAddress(); int offset = Platform.getInt(null, offsetBufferAddress + rowId * 4L); int length = Platform.getInt(null, offsetBufferAddress + (rowId + 1L) * 4L) - offset; return UTF8String.fromAddress(null, valueBufferAddress + offset, length); } else { BaseFixedWidthVector fixedWidthVector = (BaseFixedWidthVector) valueVector; int length = fixedWidthVector.getTypeWidth(); int offset = rowId * length; byte[] result = new byte[length]; Platform.copyMemory( null, valueBufferAddress + offset, result, Platform.BYTE_ARRAY_OFFSET, length); if (!isUuid) { return UTF8String.fromBytes(result); } else { return UTF8String.fromString(convertToUuid(result).toString()); } } } @Override public byte[] getBinary(int rowId) { if (isNullAt(rowId)) return null; int offset; int length; if (valueVector instanceof BaseVariableWidthVector) { BaseVariableWidthVector varWidthVector = (BaseVariableWidthVector) valueVector; long offsetBufferAddress = varWidthVector.getOffsetBuffer().memoryAddress(); offset = Platform.getInt(null, offsetBufferAddress + rowId * 4L); length = Platform.getInt(null, offsetBufferAddress + (rowId + 1L) * 4L) - offset; } else if (valueVector instanceof BaseFixedWidthVector) { BaseFixedWidthVector fixedWidthVector = (BaseFixedWidthVector) valueVector; length = fixedWidthVector.getTypeWidth(); offset = rowId * length; } else { throw new RuntimeException("Unsupported binary vector type: " + valueVector.getName()); } byte[] result = new byte[length]; Platform.copyMemory( null, valueBufferAddress + offset, result, Platform.BYTE_ARRAY_OFFSET, length); return result; } @Override public CDataDictionaryProvider getDictionaryProvider() { return null; } @Override public boolean isNullAt(int rowId) { return this.valueBufferAddress == -1 || super.isNullAt(rowId); } @Override public CometVector slice(int offset, int length) { TransferPair tp = this.valueVector.getTransferPair(this.valueVector.getAllocator()); tp.splitAndTransfer(offset, length); return new CometPlainVector(tp.getTo(), useDecimal128); } private static UUID convertToUuid(byte[] buf) { Preconditions.checkArgument(buf.length == 16, "UUID require 16 bytes"); ByteBuffer bb = ByteBuffer.wrap(buf); bb.order(ByteOrder.BIG_ENDIAN); long mostSigBits = bb.getLong(); long leastSigBits = bb.getLong(); return new UUID(mostSigBits, leastSigBits); } } ================================================ FILE: common/src/main/java/org/apache/comet/vector/CometSelectionVector.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.vector; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.IntVector; import org.apache.arrow.vector.dictionary.DictionaryProvider; import org.apache.spark.sql.vectorized.ColumnVector; import org.apache.spark.sql.vectorized.ColumnarArray; import org.apache.spark.sql.vectorized.ColumnarMap; import org.apache.spark.unsafe.types.UTF8String; /** * A zero-copy selection vector that extends CometVector. This implementation stores the original * data vector and selection indices as separate CometVectors, providing zero copy access to the the * underlying data. * *

If the original vector has values [v0, v1, v2, v3, v4, v5, v6, v7] and the selection indices * are [0, 1, 3, 4, 5, 7], then this selection vector will logically represent [v0, v1, v3, v4, v5, * v7] without actually copying the data. * *

Most of the implementations of CometVector methods are implemented for completeness. We don't * use this class except to transfer the original data and the selection indices to the native code. */ public class CometSelectionVector extends CometVector { /** The original vector containing all values */ private final CometVector values; /** * The valid indices in the values vector. This array is converted into an Arrow vector so we can * transfer the data to native in one JNI call. This is used to represent the rowid mapping used * by Iceberg */ private final int[] selectionIndices; /** * The indices vector containing selection indices. This is currently allocated by the JVM side * unlike the values vector which is allocated on the native side */ private final CometVector indices; /** * Number of selected elements. The indices array may have a length greater than this but only * numValues elements in the array are valid */ private final int numValues; /** * Creates a new selection vector from the given vector and indices. * * @param values The original vector to select from * @param indices The indices to select from the original vector * @param numValues The number of valid values in the indices array * @throws IllegalArgumentException if any index is out of bounds */ public CometSelectionVector(CometVector values, int[] indices, int numValues) { // Use the values vector's datatype, useDecimal128, and dictionary provider super(values.dataType(), values.useDecimal128); this.values = values; this.selectionIndices = indices; this.numValues = numValues; // Validate indices int originalLength = values.numValues(); for (int idx : indices) { if (idx < 0 || idx >= originalLength) { throw new IllegalArgumentException( String.format( "Index %d is out of bounds for vector of length %d", idx, originalLength)); } } // Create indices vector BufferAllocator allocator = values.getValueVector().getAllocator(); IntVector indicesVector = new IntVector("selection_indices", allocator); indicesVector.allocateNew(numValues); for (int i = 0; i < numValues; i++) { indicesVector.set(i, indices[i]); } indicesVector.setValueCount(numValues); this.indices = CometVector.getVector(indicesVector, values.useDecimal128, values.getDictionaryProvider()); } /** * Returns the index in the values vector for the given selection vector index. * * @param selectionIndex Index in the selection vector * @return The corresponding index in the original vector * @throws IndexOutOfBoundsException if selectionIndex is out of bounds */ private int getValuesIndex(int selectionIndex) { if (selectionIndex < 0 || selectionIndex >= numValues) { throw new IndexOutOfBoundsException( String.format( "Selection index %d is out of bounds for selection vector of length %d", selectionIndex, numValues)); } return indices.getInt(selectionIndex); } /** * Returns a reference to the values vector. * * @return The CometVector containing the values */ public CometVector getValues() { return values; } /** * Returns the indices vector. * * @return The CometVector containing the indices */ public CometVector getIndices() { return indices; } /** * Returns the selected indices. * * @return Array of selected indices */ private int[] getSelectedIndices() { return selectionIndices; } @Override public int numValues() { return numValues; } @Override public void setNumValues(int numValues) { // For selection vectors, we don't allow changing the number of values // as it would break the mapping between selection indices and values throw new UnsupportedOperationException("CometSelectionVector doesn't support setNumValues"); } @Override public void setNumNulls(int numNulls) { // For selection vectors, null count should be delegated to the underlying values vector // The selection doesn't change the null semantics values.setNumNulls(numNulls); } @Override public boolean hasNull() { return values.hasNull(); } @Override public int numNulls() { return values.numNulls(); } // ColumnVector method implementations - delegate to original vector with index mapping @Override public boolean isNullAt(int rowId) { return values.isNullAt(getValuesIndex(rowId)); } @Override public boolean getBoolean(int rowId) { return values.getBoolean(getValuesIndex(rowId)); } @Override public byte getByte(int rowId) { return values.getByte(getValuesIndex(rowId)); } @Override public short getShort(int rowId) { return values.getShort(getValuesIndex(rowId)); } @Override public int getInt(int rowId) { return values.getInt(getValuesIndex(rowId)); } @Override public long getLong(int rowId) { return values.getLong(getValuesIndex(rowId)); } @Override public long getLongDecimal(int rowId) { return values.getLongDecimal(getValuesIndex(rowId)); } @Override public float getFloat(int rowId) { return values.getFloat(getValuesIndex(rowId)); } @Override public double getDouble(int rowId) { return values.getDouble(getValuesIndex(rowId)); } @Override public UTF8String getUTF8String(int rowId) { return values.getUTF8String(getValuesIndex(rowId)); } @Override public byte[] getBinary(int rowId) { return values.getBinary(getValuesIndex(rowId)); } @Override public ColumnarArray getArray(int rowId) { return values.getArray(getValuesIndex(rowId)); } @Override public ColumnarMap getMap(int rowId) { return values.getMap(getValuesIndex(rowId)); } @Override public ColumnVector getChild(int ordinal) { // Return the child from the original vector return values.getChild(ordinal); } @Override public DictionaryProvider getDictionaryProvider() { return values.getDictionaryProvider(); } @Override public CometVector slice(int offset, int length) { if (offset < 0 || length < 0 || offset + length > numValues) { throw new IllegalArgumentException("Invalid slice parameters"); } // Get the current indices and slice them int[] currentIndices = getSelectedIndices(); int[] slicedIndices = new int[length]; // This is not a very efficient version of slicing, but that is // not important because we are not likely to use it. System.arraycopy(currentIndices, offset, slicedIndices, 0, length); return new CometSelectionVector(values, slicedIndices, length); } @Override public org.apache.arrow.vector.ValueVector getValueVector() { return values.getValueVector(); } @Override public void close() { // Close both the values and indices vectors values.close(); indices.close(); } } ================================================ FILE: common/src/main/java/org/apache/comet/vector/CometStructVector.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.vector; import java.util.ArrayList; import java.util.List; import org.apache.arrow.vector.*; import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.dictionary.DictionaryProvider; import org.apache.arrow.vector.util.TransferPair; import org.apache.spark.sql.vectorized.ColumnVector; /** A Comet column vector for struct type. */ public class CometStructVector extends CometDecodedVector { final List children; final DictionaryProvider dictionaryProvider; public CometStructVector( ValueVector vector, boolean useDecimal128, DictionaryProvider dictionaryProvider) { super(vector, vector.getField(), useDecimal128); StructVector structVector = ((StructVector) vector); int size = structVector.size(); List children = new ArrayList<>(); for (int i = 0; i < size; ++i) { ValueVector value = structVector.getVectorById(i); children.add(getVector(value, useDecimal128, dictionaryProvider)); } this.children = children; this.dictionaryProvider = dictionaryProvider; } @Override public ColumnVector getChild(int i) { return children.get(i); } @Override public CometVector slice(int offset, int length) { TransferPair tp = this.valueVector.getTransferPair(this.valueVector.getAllocator()); tp.splitAndTransfer(offset, length); return new CometStructVector(tp.getTo(), useDecimal128, dictionaryProvider); } } ================================================ FILE: common/src/main/java/org/apache/comet/vector/CometVector.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.vector; import java.math.BigDecimal; import java.math.BigInteger; import org.apache.arrow.vector.FixedWidthVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.dictionary.Dictionary; import org.apache.arrow.vector.dictionary.DictionaryProvider; import org.apache.arrow.vector.types.pojo.DictionaryEncoding; import org.apache.spark.sql.types.DataType; import org.apache.spark.sql.types.Decimal; import org.apache.spark.sql.types.IntegerType; import org.apache.spark.sql.vectorized.ColumnVector; import org.apache.spark.sql.vectorized.ColumnarArray; import org.apache.spark.sql.vectorized.ColumnarMap; import org.apache.spark.unsafe.Platform; import org.apache.spark.unsafe.types.UTF8String; import org.apache.comet.IcebergApi; /** Base class for all Comet column vector implementations. */ @IcebergApi public abstract class CometVector extends ColumnVector { private static final int DECIMAL_BYTE_WIDTH = 16; private final byte[] DECIMAL_BYTES = new byte[DECIMAL_BYTE_WIDTH]; protected final boolean useDecimal128; private static final long decimalValOffset; static { try { java.lang.reflect.Field unsafeField = sun.misc.Unsafe.class.getDeclaredField("theUnsafe"); unsafeField.setAccessible(true); final sun.misc.Unsafe unsafe = (sun.misc.Unsafe) unsafeField.get(null); decimalValOffset = unsafe.objectFieldOffset(Decimal.class.getDeclaredField("decimalVal")); } catch (Throwable e) { throw new RuntimeException(e); } } @IcebergApi public CometVector(DataType type, boolean useDecimal128) { super(type); this.useDecimal128 = useDecimal128; } /** * Sets the number of nulls in this vector to be 'numNulls'. This is used when the vector is * reused across batches. */ @IcebergApi public abstract void setNumNulls(int numNulls); /** * Sets the number of values (including both nulls and non-nulls) in this vector to be * 'numValues'. This is used when the vector is reused across batches. */ @IcebergApi public abstract void setNumValues(int numValues); /** Returns the number of values in this vector. */ @IcebergApi public abstract int numValues(); /** Whether the elements of this vector are of fixed length. */ public boolean isFixedLength() { return getValueVector() instanceof FixedWidthVector; } @Override public Decimal getDecimal(int i, int precision, int scale) { if (isNullAt(i)) return null; if (!useDecimal128 && precision <= Decimal.MAX_INT_DIGITS() && type instanceof IntegerType) { return createDecimal(getInt(i), precision, scale); } else if (precision <= Decimal.MAX_LONG_DIGITS()) { return createDecimal(useDecimal128 ? getLongDecimal(i) : getLong(i), precision, scale); } else { byte[] bytes = getBinaryDecimal(i); BigInteger bigInteger = new BigInteger(bytes); BigDecimal javaDecimal = new BigDecimal(bigInteger, scale); return createDecimal(javaDecimal, precision, scale); } } /** This method skips the negative scale check, otherwise the same as Decimal.createUnsafe(). */ private Decimal createDecimal(long unscaled, int precision, int scale) { Decimal dec = new Decimal(); dec.org$apache$spark$sql$types$Decimal$$longVal_$eq(unscaled); dec.org$apache$spark$sql$types$Decimal$$_precision_$eq(precision); dec.org$apache$spark$sql$types$Decimal$$_scale_$eq(scale); return dec; } /** This method skips a few checks, otherwise the same as Decimal.apply(). */ private Decimal createDecimal(BigDecimal value, int precision, int scale) { Decimal dec = new Decimal(); Platform.putObjectVolatile(dec, decimalValOffset, new scala.math.BigDecimal(value)); dec.org$apache$spark$sql$types$Decimal$$_precision_$eq(precision); dec.org$apache$spark$sql$types$Decimal$$_scale_$eq(scale); return dec; } /** * Reads a 16-byte byte array which are encoded big-endian for decimal128 into internal byte * array. */ byte[] getBinaryDecimal(int i) { return copyBinaryDecimal(i, DECIMAL_BYTES); } /** Reads a 16-byte byte array which are encoded big-endian for decimal128. */ public byte[] copyBinaryDecimal(int i, byte[] dest) { long valueBufferAddress = getValueVector().getDataBuffer().memoryAddress(); Platform.copyMemory( null, valueBufferAddress + (long) i * DECIMAL_BYTE_WIDTH, dest, Platform.BYTE_ARRAY_OFFSET, DECIMAL_BYTE_WIDTH); // Decimal is stored little-endian in Arrow, so we need to reverse the bytes here for (int j = 0, k = DECIMAL_BYTE_WIDTH - 1; j < DECIMAL_BYTE_WIDTH / 2; j++, k--) { byte tmp = dest[j]; dest[j] = dest[k]; dest[k] = tmp; } return dest; } @Override public boolean getBoolean(int rowId) { throw notImplementedException(); } @Override public byte getByte(int rowId) { throw notImplementedException(); } @Override public short getShort(int rowId) { throw notImplementedException(); } @Override public int getInt(int rowId) { throw notImplementedException(); } @Override public long getLong(int rowId) { throw notImplementedException(); } public long getLongDecimal(int rowId) { throw notImplementedException(); } @Override public float getFloat(int rowId) { throw notImplementedException(); } @Override public double getDouble(int rowId) { throw notImplementedException(); } @Override public UTF8String getUTF8String(int rowId) { throw notImplementedException(); } @Override public byte[] getBinary(int rowId) { throw notImplementedException(); } @Override public ColumnarArray getArray(int i) { throw notImplementedException(); } @Override public ColumnarMap getMap(int i) { throw notImplementedException(); } @Override public ColumnVector getChild(int i) { throw notImplementedException(); } @Override public void close() { getValueVector().close(); } public DictionaryProvider getDictionaryProvider() { throw new UnsupportedOperationException("Not implemented"); } @IcebergApi public abstract ValueVector getValueVector(); /** * Returns a zero-copying new vector that contains the values from [offset, offset + length). * * @param offset the offset of the new vector * @param length the length of the new vector * @return the new vector */ @IcebergApi public abstract CometVector slice(int offset, int length); /** * Returns a corresponding `CometVector` implementation based on the given Arrow `ValueVector`. * * @param vector Arrow `ValueVector` * @param useDecimal128 Whether to use Decimal128 for decimal column * @return `CometVector` implementation */ public static CometVector getVector( ValueVector vector, boolean useDecimal128, DictionaryProvider dictionaryProvider) { if (vector instanceof StructVector) { return new CometStructVector(vector, useDecimal128, dictionaryProvider); } else if (vector instanceof MapVector) { return new CometMapVector(vector, useDecimal128, dictionaryProvider); } else if (vector instanceof ListVector) { return new CometListVector(vector, useDecimal128, dictionaryProvider); } else { DictionaryEncoding dictionaryEncoding = vector.getField().getDictionary(); CometPlainVector cometVector = new CometPlainVector(vector, useDecimal128); if (dictionaryEncoding == null) { return cometVector; } else { Dictionary dictionary = dictionaryProvider.lookup(dictionaryEncoding.getId()); CometPlainVector dictionaryVector = new CometPlainVector(dictionary.getVector(), useDecimal128); CometDictionary cometDictionary = new CometDictionary(dictionaryVector); return new CometDictionaryVector( cometVector, cometDictionary, dictionaryProvider, useDecimal128); } } } protected static CometVector getVector(ValueVector vector, boolean useDecimal128) { return getVector(vector, useDecimal128, null); } private UnsupportedOperationException notImplementedException() { return new UnsupportedOperationException( "CometVector subclass " + this.getClass().getName() + " does not implement this method"); } } ================================================ FILE: common/src/main/resources/log4j2.properties ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # Set everything to be logged to the file target/unit-tests.log rootLogger.level = info rootLogger.appenderRef.file.ref = ${sys:test.appender:-File} appender.file.type = File appender.file.name = File appender.file.fileName = target/unit-tests.log appender.file.layout.type = PatternLayout appender.file.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n # Tests that launch java subprocesses can set the "test.appender" system property to # "console" to avoid having the child process's logs overwrite the unit test's # log file. appender.console.type = Console appender.console.name = console appender.console.target = SYSTEM_ERR appender.console.layout.type = PatternLayout appender.console.layout.pattern = %t: %m%n # Ignore messages below warning level from Jetty, because it's a bit verbose logger.jetty.name = org.sparkproject.jetty logger.jetty.level = warn ================================================ FILE: common/src/main/scala/org/apache/comet/CometConf.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet import java.util.Locale import java.util.concurrent.TimeUnit import scala.collection.mutable.ListBuffer import org.apache.spark.network.util.ByteUnit import org.apache.spark.network.util.JavaUtils import org.apache.spark.sql.comet.util.Utils import org.apache.spark.sql.internal.SQLConf import org.apache.comet.shims.ShimCometConf /** * Configurations for a Comet application. Mostly inspired by [[SQLConf]] in Spark. * * To get the value of a Comet config key from a [[SQLConf]], you can do the following: * * {{{ * CometConf.COMET_ENABLED.get * }}} * * which retrieves the config value from the thread-local [[SQLConf]] object. Alternatively, you * can also explicitly pass a [[SQLConf]] object to the `get` method. */ object CometConf extends ShimCometConf { val COMPAT_GUIDE: String = "For more information, refer to the Comet Compatibility " + "Guide (https://datafusion.apache.org/comet/user-guide/compatibility.html)" private val TUNING_GUIDE = "For more information, refer to the Comet Tuning " + "Guide (https://datafusion.apache.org/comet/user-guide/tuning.html)" private val TRACING_GUIDE = "For more information, refer to the Comet Tracing " + "Guide (https://datafusion.apache.org/comet/contributor-guide/tracing.html)" private val DEBUGGING_GUIDE = "For more information, refer to the Comet Debugging " + "Guide (https://datafusion.apache.org/comet/contributor-guide/debugging.html)" /** List of all configs that is used for generating documentation */ val allConfs = new ListBuffer[ConfigEntry[_]] private val CATEGORY_SCAN = "scan" private val CATEGORY_PARQUET = "parquet" private val CATEGORY_EXEC = "exec" private val CATEGORY_EXEC_EXPLAIN = "exec_explain" private val CATEGORY_ENABLE_EXEC = "enable_exec" private val CATEGORY_SHUFFLE = "shuffle" private val CATEGORY_TUNING = "tuning" private val CATEGORY_TESTING = "testing" def register(conf: ConfigEntry[_]): Unit = { assert(conf.category.nonEmpty, s"${conf.key} does not have a category defined") allConfs.append(conf) } def conf(key: String): ConfigBuilder = ConfigBuilder(key) val COMET_PREFIX = "spark.comet"; val COMET_EXEC_CONFIG_PREFIX: String = s"$COMET_PREFIX.exec" val COMET_EXPR_CONFIG_PREFIX: String = s"$COMET_PREFIX.expression" val COMET_OPERATOR_CONFIG_PREFIX: String = s"$COMET_PREFIX.operator" val COMET_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.enabled") .category(CATEGORY_EXEC) .doc( "Whether to enable Comet extension for Spark. When this is turned on, Spark will use " + "Comet to read Parquet data source. Note that to enable native vectorized execution, " + "both this config and `spark.comet.exec.enabled` need to be enabled.") .booleanConf .createWithEnvVarOrDefault("ENABLE_COMET", true) val COMET_NATIVE_SCAN_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.scan.enabled") .category(CATEGORY_SCAN) .doc( "Whether to enable native scans. When this is turned on, Spark will use Comet to " + "read supported data sources (currently only Parquet is supported natively). Note " + "that to enable native vectorized execution, both this config and " + "`spark.comet.exec.enabled` need to be enabled.") .booleanConf .createWithDefault(true) val COMET_NATIVE_PARQUET_WRITE_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.parquet.write.enabled") .category(CATEGORY_TESTING) .doc( "Whether to enable native Parquet write through Comet. When enabled, " + "Comet will intercept Parquet write operations and execute them natively. This " + "feature is highly experimental and only partially implemented. It should not " + "be used in production.") .booleanConf .createWithEnvVarOrDefault("ENABLE_COMET_WRITE", false) val SCAN_NATIVE_DATAFUSION = "native_datafusion" val SCAN_NATIVE_ICEBERG_COMPAT = "native_iceberg_compat" val SCAN_AUTO = "auto" val COMET_NATIVE_SCAN_IMPL: ConfigEntry[String] = conf("spark.comet.scan.impl") .category(CATEGORY_PARQUET) .doc( "The implementation of Comet's Parquet scan to use. Available scans are " + s"`$SCAN_NATIVE_DATAFUSION`, and `$SCAN_NATIVE_ICEBERG_COMPAT`. " + s"`$SCAN_NATIVE_DATAFUSION` is a fully native implementation, and " + s"`$SCAN_NATIVE_ICEBERG_COMPAT` is a hybrid implementation that supports some " + "additional features, such as row indexes and field ids. " + s"`$SCAN_AUTO` (default) chooses the best available scan based on the scan schema.") .stringConf .transform(_.toLowerCase(Locale.ROOT)) .checkValues(Set(SCAN_NATIVE_DATAFUSION, SCAN_NATIVE_ICEBERG_COMPAT, SCAN_AUTO)) .createWithEnvVarOrDefault("COMET_PARQUET_SCAN_IMPL", SCAN_AUTO) val COMET_ICEBERG_NATIVE_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.scan.icebergNative.enabled") .category(CATEGORY_SCAN) .doc( "Whether to enable native Iceberg table scan using iceberg-rust. When enabled, " + "Iceberg tables are read directly through native execution, bypassing Spark's " + "DataSource V2 API for better performance.") .booleanConf .createWithDefault(true) val COMET_ICEBERG_DATA_FILE_CONCURRENCY_LIMIT: ConfigEntry[Int] = conf("spark.comet.scan.icebergNative.dataFileConcurrencyLimit") .category(CATEGORY_SCAN) .doc( "The number of Iceberg data files to read concurrently within a single task. " + "Higher values improve throughput for tables with many small files by overlapping " + "I/O latency, but increase memory usage. Values between 2 and 8 are suggested.") .intConf .checkValue(v => v > 0, "Data file concurrency limit must be positive") .createWithDefault(1) val COMET_CSV_V2_NATIVE_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.scan.csv.v2.enabled") .category(CATEGORY_TESTING) .doc( "Whether to use the native Comet V2 CSV reader for improved performance. " + "Default: false (uses standard Spark CSV reader) " + "Experimental: Performance benefits are workload-dependent.") .booleanConf .createWithDefault(false) val COMET_RESPECT_PARQUET_FILTER_PUSHDOWN: ConfigEntry[Boolean] = conf("spark.comet.parquet.respectFilterPushdown") .category(CATEGORY_PARQUET) .doc( "Whether to respect Spark's PARQUET_FILTER_PUSHDOWN_ENABLED config. This needs to be " + "respected when running the Spark SQL test suite but the default setting " + "results in poor performance in Comet when using the new native scans, " + "disabled by default") .booleanConf .createWithDefault(false) val COMET_PARQUET_PARALLEL_IO_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.parquet.read.parallel.io.enabled") .category(CATEGORY_PARQUET) .doc( "Whether to enable Comet's parallel reader for Parquet files. The parallel reader reads " + "ranges of consecutive data in a file in parallel. It is faster for large files and " + "row groups but uses more resources.") .booleanConf .createWithDefault(true) val COMET_PARQUET_PARALLEL_IO_THREADS: ConfigEntry[Int] = conf("spark.comet.parquet.read.parallel.io.thread-pool.size") .category(CATEGORY_PARQUET) .doc("The maximum number of parallel threads the parallel reader will use in a single " + "executor. For executors configured with a smaller number of cores, use a smaller number.") .intConf .createWithDefault(16) val COMET_IO_MERGE_RANGES: ConfigEntry[Boolean] = conf("spark.comet.parquet.read.io.mergeRanges") .category(CATEGORY_PARQUET) .doc( "When enabled the parallel reader will try to merge ranges of data that are separated " + "by less than `comet.parquet.read.io.mergeRanges.delta` bytes. Longer continuous reads " + "are faster on cloud storage.") .booleanConf .createWithDefault(true) val COMET_IO_MERGE_RANGES_DELTA: ConfigEntry[Int] = conf("spark.comet.parquet.read.io.mergeRanges.delta") .category(CATEGORY_PARQUET) .doc("The delta in bytes between consecutive read ranges below which the parallel reader " + "will try to merge the ranges. The default is 8MB.") .intConf .createWithDefault(1 << 23) // 8 MB val COMET_IO_ADJUST_READRANGE_SKEW: ConfigEntry[Boolean] = conf("spark.comet.parquet.read.io.adjust.readRange.skew") .category(CATEGORY_PARQUET) .doc("In the parallel reader, if the read ranges submitted are skewed in sizes, this " + "option will cause the reader to break up larger read ranges into smaller ranges to " + "reduce the skew. This will result in a slightly larger number of connections opened to " + "the file system but may give improved performance.") .booleanConf .createWithDefault(false) val COMET_CONVERT_FROM_PARQUET_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.convert.parquet.enabled") .category(CATEGORY_EXEC) .doc( "When enabled, data from Spark (non-native) Parquet v1 and v2 scans will be converted to " + "Arrow format.") .booleanConf .createWithDefault(false) val COMET_CONVERT_FROM_JSON_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.convert.json.enabled") .category(CATEGORY_EXEC) .doc( "When enabled, data from Spark (non-native) JSON v1 and v2 scans will be converted to " + "Arrow format.") .booleanConf .createWithDefault(false) val COMET_CONVERT_FROM_CSV_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.convert.csv.enabled") .category(CATEGORY_EXEC) .doc( "When enabled, data from Spark (non-native) CSV v1 and v2 scans will be converted to " + "Arrow format.") .booleanConf .createWithDefault(false) val COMET_EXEC_ENABLED: ConfigEntry[Boolean] = conf(s"$COMET_EXEC_CONFIG_PREFIX.enabled") .category(CATEGORY_EXEC) .doc( "Whether to enable Comet native vectorized execution for Spark. This controls whether " + "Spark should convert operators into their Comet counterparts and execute them in " + "native space. Note: each operator is associated with a separate config in the " + "format of `spark.comet.exec..enabled` at the moment, and both the " + "config and this need to be turned on, in order for the operator to be executed in " + "native.") .booleanConf .createWithDefault(true) val COMET_EXEC_PROJECT_ENABLED: ConfigEntry[Boolean] = createExecEnabledConfig("project", defaultValue = true) val COMET_EXEC_FILTER_ENABLED: ConfigEntry[Boolean] = createExecEnabledConfig("filter", defaultValue = true) val COMET_EXEC_SORT_ENABLED: ConfigEntry[Boolean] = createExecEnabledConfig("sort", defaultValue = true) val COMET_EXEC_LOCAL_LIMIT_ENABLED: ConfigEntry[Boolean] = createExecEnabledConfig("localLimit", defaultValue = true) val COMET_EXEC_GLOBAL_LIMIT_ENABLED: ConfigEntry[Boolean] = createExecEnabledConfig("globalLimit", defaultValue = true) val COMET_EXEC_BROADCAST_HASH_JOIN_ENABLED: ConfigEntry[Boolean] = createExecEnabledConfig("broadcastHashJoin", defaultValue = true) val COMET_EXEC_BROADCAST_EXCHANGE_ENABLED: ConfigEntry[Boolean] = createExecEnabledConfig("broadcastExchange", defaultValue = true) val COMET_EXEC_HASH_JOIN_ENABLED: ConfigEntry[Boolean] = createExecEnabledConfig("hashJoin", defaultValue = true) val COMET_EXEC_SORT_MERGE_JOIN_ENABLED: ConfigEntry[Boolean] = createExecEnabledConfig("sortMergeJoin", defaultValue = true) val COMET_EXEC_AGGREGATE_ENABLED: ConfigEntry[Boolean] = createExecEnabledConfig("aggregate", defaultValue = true) val COMET_EXEC_COLLECT_LIMIT_ENABLED: ConfigEntry[Boolean] = createExecEnabledConfig("collectLimit", defaultValue = true) val COMET_EXEC_COALESCE_ENABLED: ConfigEntry[Boolean] = createExecEnabledConfig("coalesce", defaultValue = true) val COMET_EXEC_UNION_ENABLED: ConfigEntry[Boolean] = createExecEnabledConfig("union", defaultValue = true) val COMET_EXEC_EXPAND_ENABLED: ConfigEntry[Boolean] = createExecEnabledConfig("expand", defaultValue = true) val COMET_EXEC_EXPLODE_ENABLED: ConfigEntry[Boolean] = createExecEnabledConfig("explode", defaultValue = true) val COMET_EXEC_WINDOW_ENABLED: ConfigEntry[Boolean] = createExecEnabledConfig("window", defaultValue = true) val COMET_EXEC_TAKE_ORDERED_AND_PROJECT_ENABLED: ConfigEntry[Boolean] = createExecEnabledConfig("takeOrderedAndProject", defaultValue = true) val COMET_EXEC_LOCAL_TABLE_SCAN_ENABLED: ConfigEntry[Boolean] = createExecEnabledConfig("localTableScan", defaultValue = false) val COMET_NATIVE_COLUMNAR_TO_ROW_ENABLED: ConfigEntry[Boolean] = conf(s"$COMET_EXEC_CONFIG_PREFIX.columnarToRow.native.enabled") .category(CATEGORY_EXEC) .doc( "Whether to enable native columnar to row conversion. When enabled, Comet will use " + "native Rust code to convert Arrow columnar data to Spark UnsafeRow format instead " + "of the JVM implementation. This can improve performance for queries that need to " + "convert between columnar and row formats.") .booleanConf .createWithDefault(true) val COMET_EXEC_SORT_MERGE_JOIN_WITH_JOIN_FILTER_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.exec.sortMergeJoinWithJoinFilter.enabled") .category(CATEGORY_ENABLE_EXEC) .doc("Experimental support for Sort Merge Join with filter") .booleanConf .createWithDefault(false) val COMET_TRACING_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.tracing.enabled") .category(CATEGORY_TUNING) .doc(s"Enable fine-grained tracing of events and memory usage. $TRACING_GUIDE.") .booleanConf .createWithDefault(false) val COMET_ONHEAP_MEMORY_OVERHEAD: ConfigEntry[Long] = conf("spark.comet.memoryOverhead") .category(CATEGORY_TESTING) .doc( "The amount of additional memory to be allocated per executor process for Comet, in MiB, " + "when running Spark in on-heap mode.") .bytesConf(ByteUnit.MiB) .createWithDefault(1024) val COMET_EXEC_SHUFFLE_ENABLED: ConfigEntry[Boolean] = conf(s"$COMET_EXEC_CONFIG_PREFIX.shuffle.enabled") .category(CATEGORY_SHUFFLE) .doc( "Whether to enable Comet native shuffle. " + "Note that this requires setting `spark.shuffle.manager` to " + "`org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager`. " + "`spark.shuffle.manager` must be set before starting the Spark application and " + "cannot be changed during the application.") .booleanConf .createWithDefault(true) val COMET_SHUFFLE_DIRECT_READ_ENABLED: ConfigEntry[Boolean] = conf(s"$COMET_EXEC_CONFIG_PREFIX.shuffle.directRead.enabled") .category(CATEGORY_SHUFFLE) .doc( "When enabled, native operators that consume shuffle output will read " + "compressed shuffle blocks directly in native code, bypassing Arrow FFI. " + "Applies to both native shuffle and JVM columnar shuffle. " + "Requires spark.comet.exec.shuffle.enabled to be true.") .booleanConf .createWithDefault(true) val COMET_SHUFFLE_MODE: ConfigEntry[String] = conf(s"$COMET_EXEC_CONFIG_PREFIX.shuffle.mode") .category(CATEGORY_SHUFFLE) .doc( "This is test config to allow tests to force a particular shuffle implementation to be " + "used. Valid values are `jvm` for Columnar Shuffle, `native` for Native Shuffle, " + s"and `auto` to pick the best supported option (`native` has priority). $TUNING_GUIDE.") .internal() .stringConf .transform(_.toLowerCase(Locale.ROOT)) .checkValues(Set("native", "jvm", "auto")) .createWithDefault("auto") val COMET_EXEC_BROADCAST_FORCE_ENABLED: ConfigEntry[Boolean] = conf(s"$COMET_EXEC_CONFIG_PREFIX.broadcast.enabled") .category(CATEGORY_EXEC) .doc( "Whether to force enabling broadcasting for Comet native operators. " + "Comet broadcast feature will be enabled automatically by " + "Comet extension. But for unit tests, we need this feature to force enabling it " + "for invalid cases. So this config is only used for unit test.") .internal() .booleanConf .createWithDefault(false) val COMET_REPLACE_SMJ: ConfigEntry[Boolean] = conf(s"$COMET_EXEC_CONFIG_PREFIX.replaceSortMergeJoin") .category(CATEGORY_EXEC) .doc("Experimental feature to force Spark to replace SortMergeJoin with ShuffledHashJoin " + s"for improved performance. This feature is not stable yet. $TUNING_GUIDE.") .booleanConf .createWithDefault(false) val COMET_EXEC_SHUFFLE_WITH_HASH_PARTITIONING_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.native.shuffle.partitioning.hash.enabled") .category(CATEGORY_SHUFFLE) .doc("Whether to enable hash partitioning for Comet native shuffle.") .booleanConf .createWithDefault(true) val COMET_EXEC_SHUFFLE_WITH_RANGE_PARTITIONING_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.native.shuffle.partitioning.range.enabled") .category(CATEGORY_SHUFFLE) .doc("Whether to enable range partitioning for Comet native shuffle.") .booleanConf .createWithDefault(true) val COMET_EXEC_SHUFFLE_WITH_ROUND_ROBIN_PARTITIONING_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.native.shuffle.partitioning.roundrobin.enabled") .category(CATEGORY_SHUFFLE) .doc( "Whether to enable round robin partitioning for Comet native shuffle. " + "This is disabled by default because Comet's round-robin produces different " + "partition assignments than Spark. Spark sorts rows by their binary UnsafeRow " + "representation before assigning partitions, but Comet uses Arrow format which " + "has a different binary layout. Instead, Comet implements round-robin as hash " + "partitioning on all columns, which achieves the same goals: even distribution, " + "deterministic output (for fault tolerance), and no semantic grouping. " + "Sorted output will be identical to Spark, but unsorted row ordering may differ.") .booleanConf .createWithDefault(false) val COMET_EXEC_SHUFFLE_WITH_ROUND_ROBIN_PARTITIONING_MAX_HASH_COLUMNS: ConfigEntry[Int] = conf("spark.comet.native.shuffle.partitioning.roundrobin.maxHashColumns") .category(CATEGORY_SHUFFLE) .doc( "The maximum number of columns to hash for round robin partitioning. " + "When set to 0 (the default), all columns are hashed. " + "When set to a positive value, only the first N columns are used for hashing, " + "which can improve performance for wide tables while still providing " + "reasonable distribution.") .intConf .checkValue( v => v >= 0, "The maximum number of columns to hash for round robin partitioning must be non-negative.") .createWithDefault(0) val COMET_EXEC_SHUFFLE_COMPRESSION_CODEC: ConfigEntry[String] = conf(s"$COMET_EXEC_CONFIG_PREFIX.shuffle.compression.codec") .category(CATEGORY_SHUFFLE) .doc( "The codec of Comet native shuffle used to compress shuffle data. lz4, zstd, and " + "snappy are supported. Compression can be disabled by setting " + "spark.shuffle.compress=false.") .stringConf .checkValues(Set("zstd", "lz4", "snappy")) .createWithDefault("lz4") val COMET_EXEC_SHUFFLE_COMPRESSION_ZSTD_LEVEL: ConfigEntry[Int] = conf(s"$COMET_EXEC_CONFIG_PREFIX.shuffle.compression.zstd.level") .category(CATEGORY_SHUFFLE) .doc("The compression level to use when compressing shuffle files with zstd.") .intConf .createWithDefault(1) val COMET_COLUMNAR_SHUFFLE_ASYNC_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.columnar.shuffle.async.enabled") .category(CATEGORY_SHUFFLE) .doc("Whether to enable asynchronous shuffle for Arrow-based shuffle.") .booleanConf .createWithDefault(false) val COMET_COLUMNAR_SHUFFLE_ASYNC_THREAD_NUM: ConfigEntry[Int] = conf("spark.comet.columnar.shuffle.async.thread.num") .category(CATEGORY_SHUFFLE) .doc( "Number of threads used for Comet async columnar shuffle per shuffle task. " + "Note that more threads means more memory requirement to " + "buffer shuffle data before flushing to disk. Also, more threads may not always " + "improve performance, and should be set based on the number of cores available.") .intConf .createWithDefault(3) val COMET_COLUMNAR_SHUFFLE_ASYNC_MAX_THREAD_NUM: ConfigEntry[Int] = { conf("spark.comet.columnar.shuffle.async.max.thread.num") .category(CATEGORY_SHUFFLE) .doc("Maximum number of threads on an executor used for Comet async columnar shuffle. " + "This is the upper bound of total number of shuffle " + "threads per executor. In other words, if the number of cores * the number of shuffle " + "threads per task `spark.comet.columnar.shuffle.async.thread.num` is larger than " + "this config. Comet will use this config as the number of shuffle threads per " + "executor instead.") .intConf .createWithDefault(100) } val COMET_COLUMNAR_SHUFFLE_SPILL_THRESHOLD: ConfigEntry[Int] = conf("spark.comet.columnar.shuffle.spill.threshold") .category(CATEGORY_SHUFFLE) .doc( "Number of rows to be spilled used for Comet columnar shuffle. " + "For every configured number of rows, a new spill file will be created. " + "Higher value means more memory requirement to buffer shuffle data before " + "flushing to disk. As Comet uses columnar shuffle which is columnar format, " + "higher value usually helps to improve shuffle data compression ratio. This is " + "internal config for testing purpose or advanced tuning.") .internal() .intConf .createWithDefault(Int.MaxValue) val COMET_ONHEAP_SHUFFLE_MEMORY_FACTOR: ConfigEntry[Double] = conf("spark.comet.columnar.shuffle.memory.factor") .category(CATEGORY_TESTING) .doc("Fraction of Comet memory to be allocated per executor process for columnar shuffle " + s"when running in on-heap mode. $TUNING_GUIDE.") .doubleConf .checkValue( factor => factor > 0, "Ensure that Comet shuffle memory overhead factor is a double greater than 0") .createWithDefault(1.0) val COMET_BATCH_SIZE: ConfigEntry[Int] = conf("spark.comet.batchSize") .category(CATEGORY_TUNING) .doc("The columnar batch size, i.e., the maximum number of rows that a batch can contain.") .intConf .checkValue(v => v > 0, "Batch size must be positive") .createWithDefault(8192) val COMET_COLUMNAR_SHUFFLE_BATCH_SIZE: ConfigEntry[Int] = conf("spark.comet.columnar.shuffle.batch.size") .category(CATEGORY_SHUFFLE) .doc("Batch size when writing out sorted spill files on the native side. Note that " + "this should not be larger than batch size (i.e., `spark.comet.batchSize`). Otherwise " + "it will produce larger batches than expected in the native operator after shuffle.") .intConf .checkValue( v => v <= COMET_BATCH_SIZE.get(), "Should not be larger than batch size `spark.comet.batchSize`") .createWithDefault(8192) val COMET_SHUFFLE_WRITE_BUFFER_SIZE: ConfigEntry[Long] = conf(s"$COMET_EXEC_CONFIG_PREFIX.shuffle.writeBufferSize") .category(CATEGORY_SHUFFLE) .doc("Size of the write buffer in bytes used by the native shuffle writer when writing " + "shuffle data to disk. Larger values may improve write performance by reducing " + "the number of system calls, but will use more memory. " + "The default is 1MB which provides a good balance between performance and memory usage.") .bytesConf(ByteUnit.MiB) .checkValue(v => v > 0, "Write buffer size must be positive") .createWithDefault(1) val COMET_SHUFFLE_PREFER_DICTIONARY_RATIO: ConfigEntry[Double] = conf( "spark.comet.shuffle.preferDictionary.ratio") .category(CATEGORY_SHUFFLE) .doc( "The ratio of total values to distinct values in a string column to decide whether to " + "prefer dictionary encoding when shuffling the column. If the ratio is higher than " + "this config, dictionary encoding will be used on shuffling string column. This config " + "is effective if it is higher than 1.0. Note that this " + "config is only used when `spark.comet.exec.shuffle.mode` is `jvm`.") .doubleConf .createWithDefault(10.0) val COMET_EXCHANGE_SIZE_MULTIPLIER: ConfigEntry[Double] = conf( "spark.comet.shuffle.sizeInBytesMultiplier") .category(CATEGORY_SHUFFLE) .doc( "Comet reports smaller sizes for shuffle due to using Arrow's columnar memory format " + "and this can result in Spark choosing a different join strategy due to the estimated " + "size of the exchange being smaller. Comet will multiple sizeInBytes by this amount to " + "avoid regressions in join strategy.") .doubleConf .createWithDefault(1.0) val COMET_DPP_FALLBACK_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.dppFallback.enabled") .category(CATEGORY_EXEC) .doc("Whether to fall back to Spark for queries that use DPP.") .booleanConf .createWithDefault(true) val COMET_DEBUG_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.debug.enabled") .category(CATEGORY_EXEC) .doc( "Whether to enable debug mode for Comet. " + "When enabled, Comet will do additional checks for debugging purpose. For example, " + "validating array when importing arrays from JVM at native side. Note that these " + "checks may be expensive in performance and should only be enabled for debugging " + "purpose.") .booleanConf .createWithDefault(false) // Used on native side. Check spark_config.rs how the config is used val COMET_DEBUG_MEMORY_ENABLED: ConfigEntry[Boolean] = conf(s"$COMET_PREFIX.debug.memory") .category(CATEGORY_TESTING) .doc(s"When enabled, log all native memory pool interactions. $DEBUGGING_GUIDE.") .booleanConf .createWithDefault(false) val COMET_EXTENDED_EXPLAIN_FORMAT_VERBOSE = "verbose" val COMET_EXTENDED_EXPLAIN_FORMAT_FALLBACK = "fallback" val COMET_EXTENDED_EXPLAIN_FORMAT: ConfigEntry[String] = conf("spark.comet.explain.format") .category(CATEGORY_EXEC_EXPLAIN) .doc("Choose extended explain output. The default format of " + s"'$COMET_EXTENDED_EXPLAIN_FORMAT_VERBOSE' will provide the full query plan annotated " + "with fallback reasons as well as a summary of how much of the plan was accelerated " + s"by Comet. The format '$COMET_EXTENDED_EXPLAIN_FORMAT_FALLBACK' provides a list of " + "fallback reasons instead.") .stringConf .checkValues( Set(COMET_EXTENDED_EXPLAIN_FORMAT_VERBOSE, COMET_EXTENDED_EXPLAIN_FORMAT_FALLBACK)) .createWithDefault(COMET_EXTENDED_EXPLAIN_FORMAT_VERBOSE) val COMET_EXPLAIN_NATIVE_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.explain.native.enabled") .category(CATEGORY_EXEC_EXPLAIN) .doc( "When this setting is enabled, Comet will provide a tree representation of " + "the native query plan before execution and again after execution, with " + "metrics.") .booleanConf .createWithDefault(false) val COMET_EXPLAIN_TRANSFORMATIONS: ConfigEntry[Boolean] = conf("spark.comet.explain.rules") .category(CATEGORY_EXEC_EXPLAIN) .doc("When this setting is enabled, Comet will log all plan transformations performed " + "in physical optimizer rules. Default: false") .booleanConf .createWithDefault(false) val COMET_LOG_FALLBACK_REASONS: ConfigEntry[Boolean] = conf("spark.comet.logFallbackReasons.enabled") .category(CATEGORY_EXEC_EXPLAIN) .doc("When this setting is enabled, Comet will log warnings for all fallback reasons.") .booleanConf .createWithEnvVarOrDefault("ENABLE_COMET_LOG_FALLBACK_REASONS", false) val COMET_EXPLAIN_FALLBACK_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.explainFallback.enabled") .category(CATEGORY_EXEC_EXPLAIN) .doc( "When this setting is enabled, Comet will provide logging explaining the reason(s) " + "why a query stage cannot be executed natively. Set this to false to " + "reduce the amount of logging.") .booleanConf .createWithDefault(false) val COMET_ONHEAP_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.exec.onHeap.enabled") .category(CATEGORY_TESTING) .doc("Whether to allow Comet to run in on-heap mode. Required for running Spark SQL tests.") .booleanConf .createWithEnvVarOrDefault("ENABLE_COMET_ONHEAP", false) val COMET_OFFHEAP_MEMORY_POOL_TYPE: ConfigEntry[String] = conf("spark.comet.exec.memoryPool") .category(CATEGORY_TUNING) .doc( "The type of memory pool to be used for Comet native execution when running Spark in " + "off-heap mode. Available pool types are `greedy_unified` and `fair_unified`. " + s"$TUNING_GUIDE.") .stringConf .createWithDefault("fair_unified") val COMET_ONHEAP_MEMORY_POOL_TYPE: ConfigEntry[String] = conf( "spark.comet.exec.onHeap.memoryPool") .category(CATEGORY_TESTING) .doc( "The type of memory pool to be used for Comet native execution " + "when running Spark in on-heap mode. Available pool types are `greedy`, `fair_spill`, " + "`greedy_task_shared`, `fair_spill_task_shared`, `greedy_global`, `fair_spill_global`, " + "and `unbounded`.") .stringConf .createWithDefault("greedy_task_shared") val COMET_OFFHEAP_MEMORY_POOL_FRACTION: ConfigEntry[Double] = conf("spark.comet.exec.memoryPool.fraction") .category(CATEGORY_TUNING) .doc( "Fraction of off-heap memory pool that is available to Comet. " + "Only applies to off-heap mode. " + s"$TUNING_GUIDE.") .doubleConf .createWithDefault(1.0) val COMET_NATIVE_LOAD_REQUIRED: ConfigEntry[Boolean] = conf("spark.comet.nativeLoadRequired") .category(CATEGORY_EXEC) .doc( "Whether to require Comet native library to load successfully when Comet is enabled. " + "If not, Comet will silently fallback to Spark when it fails to load the native lib. " + "Otherwise, an error will be thrown and the Spark job will be aborted.") .booleanConf .createWithDefault(false) val COMET_EXCEPTION_ON_LEGACY_DATE_TIMESTAMP: ConfigEntry[Boolean] = conf("spark.comet.exceptionOnDatetimeRebase") .category(CATEGORY_EXEC) .doc("Whether to throw exception when seeing dates/timestamps from the legacy hybrid " + "(Julian + Gregorian) calendar. Since Spark 3, dates/timestamps were written according " + "to the Proleptic Gregorian calendar. When this is true, Comet will " + "throw exceptions when seeing these dates/timestamps that were written by Spark version " + "before 3.0. If this is false, these dates/timestamps will be read as if they were " + "written to the Proleptic Gregorian calendar and will not be rebased.") .booleanConf .createWithDefault(false) val COMET_USE_DECIMAL_128: ConfigEntry[Boolean] = conf("spark.comet.use.decimal128") .internal() .category(CATEGORY_EXEC) .doc("If true, Comet will always use 128 bits to represent a decimal value, regardless of " + "its precision. If false, Comet will use 32, 64 and 128 bits respectively depending on " + "the precision. N.B. this is NOT a user-facing config but should be inferred and set by " + "Comet itself.") .booleanConf .createWithDefault(false) val COMET_USE_LAZY_MATERIALIZATION: ConfigEntry[Boolean] = conf( "spark.comet.use.lazyMaterialization") .internal() .category(CATEGORY_PARQUET) .doc( "Whether to enable lazy materialization for Comet. When this is turned on, Comet will " + "read Parquet data source lazily for string and binary columns. For filter operations, " + "lazy materialization will improve read performance by skipping unused pages.") .booleanConf .createWithDefault(true) val COMET_SCHEMA_EVOLUTION_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.schemaEvolution.enabled") .internal() .category(CATEGORY_SCAN) .doc("Whether to enable schema evolution in Comet. For instance, promoting a integer " + "column to a long column, a float column to a double column, etc. This is automatically" + "enabled when reading from Iceberg tables.") .booleanConf .createWithDefault(COMET_SCHEMA_EVOLUTION_ENABLED_DEFAULT) val COMET_ENABLE_PARTIAL_HASH_AGGREGATE: ConfigEntry[Boolean] = conf("spark.comet.testing.aggregate.partialMode.enabled") .internal() .category(CATEGORY_TESTING) .doc("This setting is used in unit tests") .booleanConf .createWithDefault(true) val COMET_ENABLE_FINAL_HASH_AGGREGATE: ConfigEntry[Boolean] = conf("spark.comet.testing.aggregate.finalMode.enabled") .internal() .category(CATEGORY_TESTING) .doc("This setting is used in unit tests") .booleanConf .createWithDefault(true) val COMET_SPARK_TO_ARROW_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.sparkToColumnar.enabled") .category(CATEGORY_EXEC) .doc("Whether to enable Spark to Arrow columnar conversion. When this is turned on, " + "Comet will convert operators in " + "`spark.comet.sparkToColumnar.supportedOperatorList` into Arrow columnar format before " + "processing.") .booleanConf .createWithDefault(false) val COMET_SPARK_TO_ARROW_SUPPORTED_OPERATOR_LIST: ConfigEntry[Seq[String]] = conf("spark.comet.sparkToColumnar.supportedOperatorList") .category(CATEGORY_EXEC) .doc("A comma-separated list of operators that will be converted to Arrow columnar " + s"format when `${COMET_SPARK_TO_ARROW_ENABLED.key}` is true.") .stringConf .toSequence .createWithDefault(Seq("Range,InMemoryTableScan,RDDScan")) val COMET_CASE_CONVERSION_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.caseConversion.enabled") .category(CATEGORY_EXEC) .doc("Java uses locale-specific rules when converting strings to upper or lower case and " + "Rust does not, so we disable upper and lower by default.") .booleanConf .createWithDefault(false) val COMET_PARQUET_UNSIGNED_SMALL_INT_CHECK: ConfigEntry[Boolean] = conf("spark.comet.scan.unsignedSmallIntSafetyCheck") .category(CATEGORY_SCAN) .doc( "Parquet files may contain unsigned 8-bit integers (UINT_8) which Spark maps to " + "ShortType. When this config is true (default), Comet falls back to Spark for " + "ShortType columns because we cannot distinguish signed INT16 (safe) from unsigned " + "UINT_8 (may produce different results). Set to false to allow native execution of " + "ShortType columns if you know your data does not contain unsigned UINT_8 columns " + s"from improperly encoded Parquet files. $COMPAT_GUIDE.") .booleanConf .createWithDefault(true) val COMET_EXEC_STRICT_FLOATING_POINT: ConfigEntry[Boolean] = conf("spark.comet.exec.strictFloatingPoint") .category(CATEGORY_EXEC) .doc( "When enabled, fall back to Spark for floating-point operations that may differ from " + s"Spark, such as when comparing or sorting -0.0 and 0.0. $COMPAT_GUIDE.") .booleanConf .createWithDefault(false) val COMET_METRICS_UPDATE_INTERVAL: ConfigEntry[Long] = conf("spark.comet.metrics.updateInterval") .category(CATEGORY_EXEC) .doc("The interval in milliseconds to update metrics. If interval is negative," + " metrics will be updated upon task completion.") .longConf .createWithDefault(3000L) val COMET_METRICS_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.metrics.enabled") .category(CATEGORY_EXEC) .doc( "Whether to enable Comet metrics reporting through Spark's external monitoring system. " + "When enabled, Comet exposes metrics such as native operators, Spark operators, " + "queries planned, transitions, and acceleration ratio. These metrics can be " + "visualized through tools like Grafana when a metrics sink (e.g., Prometheus) is " + "configured. Disabled by default because Spark plan traversal adds overhead and " + "metrics require a sink to be useful. " + "This config must be set before the SparkSession is created to take effect.") .booleanConf .createWithDefault(false) val COMET_LIBHDFS_SCHEMES_KEY = "fs.comet.libhdfs.schemes" val COMET_LIBHDFS_SCHEMES: OptionalConfigEntry[String] = conf(s"spark.hadoop.$COMET_LIBHDFS_SCHEMES_KEY") .category(CATEGORY_SCAN) .doc("Defines filesystem schemes (e.g., hdfs, webhdfs) that the native side accesses " + "via libhdfs, separated by commas. Valid only when built with hdfs feature enabled.") .stringConf .createOptional // Used on native side. Check spark_config.rs how the config is used val COMET_MAX_TEMP_DIRECTORY_SIZE: ConfigEntry[Long] = conf("spark.comet.maxTempDirectorySize") .category(CATEGORY_EXEC) .doc("The maximum amount of data (in bytes) stored inside the temporary directories.") .bytesConf(ByteUnit.BYTE) .createWithDefault(100L * 1024 * 1024 * 1024) // 100 GB val COMET_RESPECT_DATAFUSION_CONFIGS: ConfigEntry[Boolean] = conf(s"$COMET_EXEC_CONFIG_PREFIX.respectDataFusionConfigs") .category(CATEGORY_TESTING) .doc( "Development and testing configuration option to allow DataFusion configs set in " + "Spark configuration settings starting with `spark.comet.datafusion.` to be passed " + "into native execution.") .booleanConf .createWithDefault(false) val COMET_STRICT_TESTING: ConfigEntry[Boolean] = conf(s"$COMET_PREFIX.testing.strict") .category(CATEGORY_TESTING) .doc("Experimental option to enable strict testing, which will fail tests that could be " + "more comprehensive, such as checking for a specific fallback reason.") .booleanConf .createWithEnvVarOrDefault("ENABLE_COMET_STRICT_TESTING", false) val COMET_OPERATOR_DATA_WRITING_COMMAND_ALLOW_INCOMPAT: ConfigEntry[Boolean] = createOperatorIncompatConfig("DataWritingCommandExec") /** Create a config to enable a specific operator */ private def createExecEnabledConfig( exec: String, defaultValue: Boolean, notes: Option[String] = None): ConfigEntry[Boolean] = { conf(s"$COMET_EXEC_CONFIG_PREFIX.$exec.enabled") .category(CATEGORY_ENABLE_EXEC) .doc( s"Whether to enable $exec by default." + notes .map(s => s" $s.") .getOrElse("")) .booleanConf .createWithDefault(defaultValue) } /** * Converts a config key to a valid environment variable name. Example: * "spark.comet.operator.DataWritingCommandExec.allowIncompatible" -> * "SPARK_COMET_OPERATOR_DATAWRITINGCOMMANDEXEC_ALLOWINCOMPATIBLE" */ private def configKeyToEnvVar(configKey: String): String = configKey.toUpperCase(Locale.ROOT).replace('.', '_') private def createOperatorIncompatConfig(name: String): ConfigEntry[Boolean] = { val configKey = getOperatorAllowIncompatConfigKey(name) val envVar = configKeyToEnvVar(configKey) conf(configKey) .category(CATEGORY_EXEC) .doc(s"Whether to allow incompatibility for operator: $name. " + s"False by default. Can be overridden with $envVar env variable") .booleanConf .createWithEnvVarOrDefault(envVar, false) } def isExprEnabled(name: String, conf: SQLConf = SQLConf.get): Boolean = { getBooleanConf(getExprEnabledConfigKey(name), defaultValue = true, conf) } def getExprEnabledConfigKey(name: String): String = { s"${CometConf.COMET_EXPR_CONFIG_PREFIX}.$name.enabled" } def isExprAllowIncompat(name: String, conf: SQLConf = SQLConf.get): Boolean = { getBooleanConf(getExprAllowIncompatConfigKey(name), defaultValue = false, conf) } def getExprAllowIncompatConfigKey(name: String): String = { s"${CometConf.COMET_EXPR_CONFIG_PREFIX}.$name.allowIncompatible" } def getExprAllowIncompatConfigKey(exprClass: Class[_]): String = { s"${CometConf.COMET_EXPR_CONFIG_PREFIX}.${exprClass.getSimpleName}.allowIncompatible" } def isOperatorAllowIncompat(name: String, conf: SQLConf = SQLConf.get): Boolean = { getBooleanConf(getOperatorAllowIncompatConfigKey(name), defaultValue = false, conf) } def getOperatorAllowIncompatConfigKey(name: String): String = { s"${CometConf.COMET_OPERATOR_CONFIG_PREFIX}.$name.allowIncompatible" } def getOperatorAllowIncompatConfigKey(exprClass: Class[_]): String = { s"${CometConf.COMET_OPERATOR_CONFIG_PREFIX}.${exprClass.getSimpleName}.allowIncompatible" } def getBooleanConf(name: String, defaultValue: Boolean, conf: SQLConf): Boolean = { conf.getConfString(name, defaultValue.toString).toLowerCase(Locale.ROOT) == "true" } } object ConfigHelpers { def toNumber[T](s: String, converter: String => T, key: String, configType: String): T = { try { converter(s.trim) } catch { case _: NumberFormatException => throw new IllegalArgumentException(s"$key should be $configType, but was $s") } } def toBoolean(s: String, key: String): Boolean = { try { s.trim.toBoolean } catch { case _: IllegalArgumentException => throw new IllegalArgumentException(s"$key should be boolean, but was $s") } } def stringToSeq[T](str: String, converter: String => T): Seq[T] = { Utils.stringToSeq(str).map(converter) } def seqToString[T](v: Seq[T], stringConverter: T => String): String = { v.map(stringConverter).mkString(",") } def timeFromString(str: String, unit: TimeUnit): Long = JavaUtils.timeStringAs(str, unit) def timeToString(v: Long, unit: TimeUnit): String = TimeUnit.MILLISECONDS.convert(v, unit) + "ms" def byteFromString(str: String, unit: ByteUnit): Long = { val (input, multiplier) = if (str.nonEmpty && str.charAt(0) == '-') { (str.substring(1), -1) } else { (str, 1) } multiplier * JavaUtils.byteStringAs(input, unit) } def byteToString(v: Long, unit: ByteUnit): String = unit.convertTo(v, ByteUnit.BYTE) + "b" } private class TypedConfigBuilder[T]( val parent: ConfigBuilder, val converter: String => T, val stringConverter: T => String) { import ConfigHelpers._ def this(parent: ConfigBuilder, converter: String => T) = { this(parent, converter, Option(_).map(_.toString).orNull) } /** Apply a transformation to the user-provided values of the config entry. */ def transform(fn: T => T): TypedConfigBuilder[T] = { new TypedConfigBuilder(parent, s => fn(converter(s)), stringConverter) } /** Checks if the user-provided value for the config matches the validator. */ def checkValue(validator: T => Boolean, errorMsg: String): TypedConfigBuilder[T] = { transform { v => if (!validator(v)) { throw new IllegalArgumentException(s"'$v' in ${parent.key} is invalid. $errorMsg") } v } } /** Check that user-provided values for the config match a pre-defined set. */ def checkValues(validValues: Set[T]): TypedConfigBuilder[T] = { transform { v => if (!validValues.contains(v)) { throw new IllegalArgumentException( s"The value of ${parent.key} should be one of ${validValues.mkString(", ")}, but was $v") } v } } /** Turns the config entry into a sequence of values of the underlying type. */ def toSequence: TypedConfigBuilder[Seq[T]] = { new TypedConfigBuilder(parent, stringToSeq(_, converter), seqToString(_, stringConverter)) } /** Creates a [[ConfigEntry]] that does not have a default value. */ def createOptional: OptionalConfigEntry[T] = { val conf = new OptionalConfigEntry[T]( parent.key, converter, stringConverter, parent._doc, parent._category, parent._public, parent._version) CometConf.register(conf) conf } /** Creates a [[ConfigEntry]] that has a default value. */ def createWithDefault(default: T): ConfigEntry[T] = { val transformedDefault = converter(stringConverter(default)) val conf = new ConfigEntryWithDefault[T]( parent.key, transformedDefault, converter, stringConverter, parent._doc, parent._category, parent._public, parent._version) CometConf.register(conf) conf } /** * Creates a [[ConfigEntry]] that has a default value, with support for environment variable * override. * * The value is resolved in the following priority order: * 1. Spark config value (if set) 2. Environment variable value (if set) 3. Default value * * @param envVar * The environment variable name to check for override value * @param default * The default value to use if neither config nor env var is set * @return * A ConfigEntry with environment variable support */ def createWithEnvVarOrDefault(envVar: String, default: T): ConfigEntry[T] = { val transformedDefault = converter(sys.env.getOrElse(envVar, stringConverter(default))) val conf = new ConfigEntryWithDefault[T]( parent.key, transformedDefault, converter, stringConverter, parent._doc, parent._category, parent._public, parent._version, Some(envVar)) CometConf.register(conf) conf } } abstract class ConfigEntry[T]( val key: String, val valueConverter: String => T, val stringConverter: T => String, val doc: String, val category: String, val isPublic: Boolean, val version: String) { /** * Retrieves the config value from the given [[SQLConf]]. */ def get(conf: SQLConf): T /** * Retrieves the config value from the current thread-local [[SQLConf]] * * @return */ def get(): T = get(SQLConf.get) def defaultValue: Option[T] = None def defaultValueString: String /** * The environment variable name that can override this config's default value, if applicable. */ def envVar: Option[String] = None override def toString: String = { s"ConfigEntry(key=$key, defaultValue=$defaultValueString, doc=$doc, " + s"public=$isPublic, version=$version)" } } private[comet] class ConfigEntryWithDefault[T]( key: String, _defaultValue: T, valueConverter: String => T, stringConverter: T => String, doc: String, category: String, isPublic: Boolean, version: String, _envVar: Option[String] = None) extends ConfigEntry(key, valueConverter, stringConverter, doc, category, isPublic, version) { override def defaultValue: Option[T] = Some(_defaultValue) override def defaultValueString: String = stringConverter(_defaultValue) override def envVar: Option[String] = _envVar def get(conf: SQLConf): T = { val tmp = conf.getConfString(key, null) if (tmp == null) { _defaultValue } else { valueConverter(tmp) } } } private[comet] class OptionalConfigEntry[T]( key: String, val rawValueConverter: String => T, val rawStringConverter: T => String, doc: String, category: String, isPublic: Boolean, version: String) extends ConfigEntry[Option[T]]( key, s => Some(rawValueConverter(s)), v => v.map(rawStringConverter).orNull, doc, category, isPublic, version) { override def defaultValueString: String = ConfigEntry.UNDEFINED override def get(conf: SQLConf): Option[T] = { Option(conf.getConfString(key, null)).map(rawValueConverter) } } private[comet] case class ConfigBuilder(key: String) { import ConfigHelpers._ var _public = true var _doc = "" var _version = "" var _category = "" def internal(): ConfigBuilder = { _public = false this } def doc(s: String): ConfigBuilder = { _doc = s this } def category(s: String): ConfigBuilder = { _category = s this } def version(v: String): ConfigBuilder = { _version = v this } def intConf: TypedConfigBuilder[Int] = { new TypedConfigBuilder(this, toNumber(_, _.toInt, key, "int")) } def longConf: TypedConfigBuilder[Long] = { new TypedConfigBuilder(this, toNumber(_, _.toLong, key, "long")) } def doubleConf: TypedConfigBuilder[Double] = { new TypedConfigBuilder(this, toNumber(_, _.toDouble, key, "double")) } def booleanConf: TypedConfigBuilder[Boolean] = { new TypedConfigBuilder(this, toBoolean(_, key)) } def stringConf: TypedConfigBuilder[String] = { new TypedConfigBuilder(this, v => v) } def timeConf(unit: TimeUnit): TypedConfigBuilder[Long] = { new TypedConfigBuilder(this, timeFromString(_, unit), timeToString(_, unit)) } def bytesConf(unit: ByteUnit): TypedConfigBuilder[Long] = { new TypedConfigBuilder(this, byteFromString(_, unit), byteToString(_, unit)) } } private object ConfigEntry { val UNDEFINED = "" } ================================================ FILE: common/src/main/scala/org/apache/comet/Constants.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet object Constants { val COMET_CONF_DIR_ENV = "COMET_CONF_DIR" val LOG_CONF_PATH = "comet.log.file.path" val LOG_CONF_NAME = "log4rs.yaml" val LOG_LEVEL_ENV = "COMET_LOG_LEVEL" } ================================================ FILE: common/src/main/scala/org/apache/comet/objectstore/NativeConfig.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.objectstore import java.net.URI import java.util.Locale import org.apache.commons.lang3.StringUtils import org.apache.hadoop.conf.Configuration import org.apache.comet.CometConf.COMET_LIBHDFS_SCHEMES_KEY object NativeConfig { private val objectStoreConfigPrefixes = Map( // Amazon S3 configurations "s3" -> Seq("fs.s3a."), "s3a" -> Seq("fs.s3a."), // Google Cloud Storage configurations "gs" -> Seq("fs.gs."), // Azure Blob Storage configurations (can use both prefixes) "wasb" -> Seq("fs.azure.", "fs.wasb."), "wasbs" -> Seq("fs.azure.", "fs.wasb."), // Azure Data Lake Storage Gen2 configurations "abfs" -> Seq("fs.abfs."), // Azure Data Lake Storage Gen2 secure configurations (can use both prefixes) "abfss" -> Seq("fs.abfss.", "fs.abfs.")) /** * Extract object store configurations from Hadoop configuration for native DataFusion usage. * This includes S3, GCS, Azure and other cloud storage configurations. * * This method extracts all configurations with supported prefixes, automatically capturing both * global configurations (e.g., fs.s3a.access.key) and per-bucket configurations (e.g., * fs.s3a.bucket.{bucket-name}.access.key). The native code will prioritize per-bucket * configurations over global ones when both are present. * * The configurations are passed to the native code which uses object_store's parse_url_opts for * consistent and standardized cloud storage support across all providers. */ def extractObjectStoreOptions(hadoopConf: Configuration, uri: URI): Map[String, String] = { val scheme = Option(uri.getScheme).map(_.toLowerCase(Locale.ROOT)).getOrElse("file") import scala.jdk.CollectionConverters._ val options = scala.collection.mutable.Map[String, String]() // The schemes will use libhdfs val libhdfsSchemes = hadoopConf.get(COMET_LIBHDFS_SCHEMES_KEY) if (StringUtils.isNotBlank(libhdfsSchemes)) { options(COMET_LIBHDFS_SCHEMES_KEY) = libhdfsSchemes } // Get prefixes for this scheme, return early if none found val prefixes = objectStoreConfigPrefixes.get(scheme) if (prefixes.isEmpty) { return options.toMap } // Extract all configurations that match the object store prefixes hadoopConf.iterator().asScala.foreach { entry => val key = entry.getKey val value = entry.getValue // Check if key starts with any of the prefixes for this scheme if (prefixes.get.exists(prefix => key.startsWith(prefix))) { options(key) = value } } options.toMap } } ================================================ FILE: common/src/main/scala/org/apache/comet/package.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache import java.util.Properties import org.apache.arrow.memory.RootAllocator import org.apache.spark.internal.Logging package object comet { /** * The root allocator for Comet execution. Because Arrow Java memory management is based on * reference counting, exposed arrays increase the reference count of the underlying buffers. * Until the reference count is zero, the memory will not be released. If the consumer side is * finished later than the close of the allocator, the allocator will think the memory is * leaked. To avoid this, we use a single allocator for the whole execution process. */ val CometArrowAllocator = new RootAllocator(Long.MaxValue) /** * Provides access to build information about the Comet libraries. This will be used by the * benchmarking software to provide the source revision and repository. In addition, the build * information is included to aid in future debugging efforts for releases. */ private object CometBuildInfo extends Logging { private val GIT_INFO_PROPS_FILENAME = "comet-git-info.properties" val props: Properties = { val props = new Properties() val resourceStream = Thread .currentThread() .getContextClassLoader .getResourceAsStream(GIT_INFO_PROPS_FILENAME) if (resourceStream != null) { try { props.load(resourceStream) } catch { case e: Exception => logError(s"Error loading properties from $GIT_INFO_PROPS_FILENAME", e) } finally { if (resourceStream != null) { try { resourceStream.close() } catch { case e: Exception => logError("Error closing Comet build info resource stream", e) } } } } else { logWarning(s"Could not find $GIT_INFO_PROPS_FILENAME") } props } } private def getProp(name: String): String = { CometBuildInfo.props.getProperty(name, "") } val COMET_VERSION: String = getProp("git.build.version") val COMET_BRANCH: String = getProp("git.branch") val COMET_REVISION: String = getProp("git.commit.id.full") val COMET_BUILD_USER_EMAIL: String = getProp("git.build.user.name") val COMET_BUILD_USER_NAME: String = getProp("git.build.user.email") val COMET_REPO_URL: String = getProp("git.remote.origin.url") val COMET_BUILD_TIMESTAMP: String = getProp("git.build.time") } ================================================ FILE: common/src/main/scala/org/apache/comet/parquet/CometParquetUtils.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.parquet import org.apache.hadoop.conf.Configuration import org.apache.parquet.crypto.DecryptionPropertiesFactory import org.apache.parquet.crypto.keytools.{KeyToolkit, PropertiesDrivenCryptoFactory} import org.apache.spark.sql.internal.SQLConf object CometParquetUtils { private val PARQUET_FIELD_ID_WRITE_ENABLED = "spark.sql.parquet.fieldId.write.enabled" private val PARQUET_FIELD_ID_READ_ENABLED = "spark.sql.parquet.fieldId.read.enabled" private val IGNORE_MISSING_PARQUET_FIELD_ID = "spark.sql.parquet.fieldId.read.ignoreMissing" // Map of encryption configuration key-value pairs that, if present, are only supported with // these specific values. Generally, these are the default values that won't be present, // but if they are present we want to check them. private val SUPPORTED_ENCRYPTION_CONFIGS: Map[String, Set[String]] = Map( // https://github.com/apache/arrow-rs/blob/main/parquet/src/encryption/ciphers.rs#L21 KeyToolkit.DATA_KEY_LENGTH_PROPERTY_NAME -> Set(KeyToolkit.DATA_KEY_LENGTH_DEFAULT.toString), KeyToolkit.KEK_LENGTH_PROPERTY_NAME -> Set(KeyToolkit.KEK_LENGTH_DEFAULT.toString), // https://github.com/apache/arrow-rs/blob/main/parquet/src/file/metadata/parser.rs#L494 PropertiesDrivenCryptoFactory.ENCRYPTION_ALGORITHM_PROPERTY_NAME -> Set("AES_GCM_V1")) def writeFieldId(conf: SQLConf): Boolean = conf.getConfString(PARQUET_FIELD_ID_WRITE_ENABLED, "false").toBoolean def writeFieldId(conf: Configuration): Boolean = conf.getBoolean(PARQUET_FIELD_ID_WRITE_ENABLED, false) def readFieldId(conf: SQLConf): Boolean = conf.getConfString(PARQUET_FIELD_ID_READ_ENABLED, "false").toBoolean def ignoreMissingIds(conf: SQLConf): Boolean = conf.getConfString(IGNORE_MISSING_PARQUET_FIELD_ID, "false").toBoolean /** * Checks if the given Hadoop configuration contains any unsupported encryption settings. * * @param hadoopConf * The Hadoop configuration to check * @return * true if all encryption configurations are supported, false if any unsupported config is * found */ def isEncryptionConfigSupported(hadoopConf: Configuration): Boolean = { // Check configurations that, if present, can only have specific allowed values val supportedListCheck = SUPPORTED_ENCRYPTION_CONFIGS.forall { case (configKey, supportedValues) => val configValue = Option(hadoopConf.get(configKey)) configValue match { case Some(value) => supportedValues.contains(value) case None => true // Config not set, so it's supported } } supportedListCheck } def encryptionEnabled(hadoopConf: Configuration): Boolean = { // TODO: Are there any other properties to check? val encryptionKeys = Seq( DecryptionPropertiesFactory.CRYPTO_FACTORY_CLASS_PROPERTY_NAME, KeyToolkit.KMS_CLIENT_CLASS_PROPERTY_NAME) encryptionKeys.exists(key => Option(hadoopConf.get(key)).exists(_.nonEmpty)) } } ================================================ FILE: common/src/main/scala/org/apache/comet/parquet/CometReaderThreadPool.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.parquet import java.util.concurrent.{Executors, ExecutorService, ThreadFactory} import java.util.concurrent.atomic.AtomicLong abstract class CometReaderThreadPool { private var threadPool: Option[ExecutorService] = None protected def threadNamePrefix: String private def initThreadPool(maxThreads: Int): ExecutorService = synchronized { if (threadPool.isEmpty) { val threadFactory: ThreadFactory = new ThreadFactory() { private val defaultThreadFactory = Executors.defaultThreadFactory val count = new AtomicLong(0) override def newThread(r: Runnable): Thread = { val thread = defaultThreadFactory.newThread(r) thread.setName(s"${threadNamePrefix}_${count.getAndIncrement()}") thread.setDaemon(true) thread } } val threadPoolExecutor = Executors.newFixedThreadPool(maxThreads, threadFactory) threadPool = Some(threadPoolExecutor) } threadPool.get } def getOrCreateThreadPool(numThreads: Int): ExecutorService = { threadPool.getOrElse(initThreadPool(numThreads)) } } // Thread pool used by the Parquet parallel reader object CometFileReaderThreadPool extends CometReaderThreadPool { override def threadNamePrefix: String = "file_reader_thread" } ================================================ FILE: common/src/main/scala/org/apache/comet/vector/NativeUtil.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.vector import scala.collection.mutable import org.apache.arrow.c.{ArrowArray, ArrowImporter, ArrowSchema, CDataDictionaryProvider, Data} import org.apache.arrow.vector.VectorSchemaRoot import org.apache.arrow.vector.dictionary.DictionaryProvider import org.apache.spark.SparkException import org.apache.spark.sql.comet.util.Utils import org.apache.spark.sql.vectorized.ColumnarBatch import org.apache.comet.CometArrowAllocator /** * Provides functionality for importing Arrow vectors from native code and wrapping them as * CometVectors. * * Also provides functionality for exporting Comet columnar batches to native code. * * Each instance of NativeUtil creates an instance of CDataDictionaryProvider (a * DictionaryProvider that is used in C Data Interface for imports). * * NativeUtil must be closed after use to release resources in the dictionary provider. */ class NativeUtil { import Utils._ /** Use the global allocator */ private val allocator = CometArrowAllocator /** ArrowImporter does not hold any state and does not need to be closed */ private val importer = new ArrowImporter(allocator) /** * Dictionary provider to use for the lifetime of this instance of NativeUtil. The dictionary * provider is closed when NativeUtil is closed. */ private val dictionaryProvider: CDataDictionaryProvider = new CDataDictionaryProvider /** * Allocates Arrow structs for the given number of columns. * * @param numCols * the number of columns * @return * a pair of Arrow arrays and Arrow schemas */ def allocateArrowStructs(numCols: Int): (Array[ArrowArray], Array[ArrowSchema]) = { val arrays = new Array[ArrowArray](numCols) val schemas = new Array[ArrowSchema](numCols) (0 until numCols).foreach { index => val arrowSchema = ArrowSchema.allocateNew(allocator) val arrowArray = ArrowArray.allocateNew(allocator) arrays(index) = arrowArray schemas(index) = arrowSchema } (arrays, schemas) } /** * Exports a ColumnarBatch to Arrow FFI and returns the memory addresses. * * This is a convenience method that allocates Arrow structs, exports the batch, and returns * just the memory addresses (without exposing the Arrow types). * * @param batch * the columnar batch to export * @return * a tuple of (array addresses, schema addresses, number of rows) */ def exportBatchToAddresses(batch: ColumnarBatch): (Array[Long], Array[Long], Int) = { val numCols = batch.numCols() val (arrays, schemas) = allocateArrowStructs(numCols) val arrayAddrs = arrays.map(_.memoryAddress()) val schemaAddrs = schemas.map(_.memoryAddress()) val numRows = exportBatch(arrayAddrs, schemaAddrs, batch) (arrayAddrs, schemaAddrs, numRows) } /** * Exports a Comet `ColumnarBatch` into a list of memory addresses that can be consumed by the * native execution. * * @param batch * the input Comet columnar batch * @return * an exported batches object containing an array containing number of rows + pairs of memory * addresses in the format of (address of Arrow array, address of Arrow schema) */ def exportBatch( arrayAddrs: Array[Long], schemaAddrs: Array[Long], batch: ColumnarBatch): Int = { val numRows = mutable.ArrayBuffer.empty[Int] (0 until batch.numCols()).foreach { index => batch.column(index) match { case selectionVector: CometSelectionVector => // For CometSelectionVector, export only the values vector val valuesVector = selectionVector.getValues val valueVector = valuesVector.getValueVector // Use the selection vector's logical row count numRows += selectionVector.numValues() val provider = if (valueVector.getField.getDictionary != null) { valuesVector.getDictionaryProvider } else { null } // The array and schema structures are allocated by native side. // Don't need to deallocate them here. val arrowSchema = ArrowSchema.wrap(schemaAddrs(index)) val arrowArray = ArrowArray.wrap(arrayAddrs(index)) Data.exportVector( allocator, getFieldVector(valueVector, "export"), provider, arrowArray, arrowSchema) case a: CometVector => val valueVector = a.getValueVector numRows += valueVector.getValueCount val provider = if (valueVector.getField.getDictionary != null) { a.getDictionaryProvider } else { null } // The array and schema structures are allocated by native side. // Don't need to deallocate them here. val arrowSchema = ArrowSchema.wrap(schemaAddrs(index)) val arrowArray = ArrowArray.wrap(arrayAddrs(index)) Data.exportVector( allocator, getFieldVector(valueVector, "export"), provider, arrowArray, arrowSchema) case c => throw new SparkException( "Comet execution only takes Arrow Arrays, but got " + s"${c.getClass}") } } if (numRows.distinct.length > 1) { throw new SparkException( s"Number of rows in each column should be the same, but got [${numRows.distinct}]") } // `ColumnarBatch.numRows` might return a different number than the actual number of rows in // the Arrow arrays. For example, Iceberg column reader will skip deleted rows internally in // its `CometVector` implementation. The `ColumnarBatch` returned by the reader will report // logical number of rows which is less than actual number of rows due to row deletion. // Similarly, CometSelectionVector represents a different number of logical rows than the // underlying vector. numRows.headOption.getOrElse(batch.numRows()) } /** * Exports a single CometVector to native side. * * @param vector * The CometVector to export * @param arrayAddr * The address of the ArrowArray structure * @param schemaAddr * The address of the ArrowSchema structure */ def exportSingleVector(vector: CometVector, arrayAddr: Long, schemaAddr: Long): Unit = { val valueVector = vector.getValueVector val provider = if (valueVector.getField.getDictionary != null) { vector.getDictionaryProvider } else { null } val arrowSchema = ArrowSchema.wrap(schemaAddr) val arrowArray = ArrowArray.wrap(arrayAddr) Data.exportVector( allocator, getFieldVector(valueVector, "export"), provider, arrowArray, arrowSchema) } /** * Gets the next batch from native execution. * * @param numOutputCols * The number of output columns * @param func * The function to call to get the next batch * @return * The number of row of the next batch, or None if there are no more batches */ def getNextBatch( numOutputCols: Int, func: (Array[Long], Array[Long]) => Long): Option[ColumnarBatch] = { val (arrays, schemas) = allocateArrowStructs(numOutputCols) val arrayAddrs = arrays.map(_.memoryAddress()) val schemaAddrs = schemas.map(_.memoryAddress()) val result = func(arrayAddrs, schemaAddrs) result match { case -1 => // EOF None case numRows => val cometVectors = importVector(arrays, schemas) Some(new ColumnarBatch(cometVectors.toArray, numRows.toInt)) } } /** * Imports a list of Arrow addresses from native execution, and return a list of Comet vectors. * * @param arrays * a list of Arrow array * @param schemas * a list of Arrow schema * @return * a list of Comet vectors */ def importVector(arrays: Array[ArrowArray], schemas: Array[ArrowSchema]): Seq[CometVector] = { val arrayVectors = mutable.ArrayBuffer.empty[CometVector] (0 until arrays.length).foreach { i => val arrowSchema = schemas(i) val arrowArray = arrays(i) // Native execution should always have 'useDecimal128' set to true since it doesn't support // other cases. arrayVectors += CometVector.getVector( importer.importVector(arrowArray, arrowSchema, dictionaryProvider), true, dictionaryProvider) } arrayVectors.toSeq } /** * Takes zero-copy slices of the input batch with given start index and maximum number of rows. * * @param batch * Input batch * @param startIndex * Start index of the slice * @param maxNumRows * Maximum number of rows in the slice * @return * A new batch with the sliced vectors */ def takeRows(batch: ColumnarBatch, startIndex: Int, maxNumRows: Int): ColumnarBatch = { val arrayVectors = mutable.ArrayBuffer.empty[CometVector] for (i <- 0 until batch.numCols()) { val column = batch.column(i).asInstanceOf[CometVector] arrayVectors += column.slice(startIndex, maxNumRows) } new ColumnarBatch(arrayVectors.toArray, maxNumRows) } def close(): Unit = { // closing the dictionary provider also closes the dictionary arrays dictionaryProvider.close() } } object NativeUtil { def rootAsBatch(arrowRoot: VectorSchemaRoot): ColumnarBatch = { rootAsBatch(arrowRoot, null) } def rootAsBatch(arrowRoot: VectorSchemaRoot, provider: DictionaryProvider): ColumnarBatch = { val vectors = (0 until arrowRoot.getFieldVectors.size()).map { i => val vector = arrowRoot.getFieldVectors.get(i) // Native shuffle always uses decimal128. CometVector.getVector(vector, true, provider) } new ColumnarBatch(vectors.toArray, arrowRoot.getRowCount) } } ================================================ FILE: common/src/main/scala/org/apache/comet/vector/StreamReader.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.vector import java.nio.channels.ReadableByteChannel import org.apache.arrow.vector.VectorSchemaRoot import org.apache.arrow.vector.ipc.{ArrowStreamReader, ReadChannel} import org.apache.arrow.vector.ipc.message.MessageChannelReader import org.apache.spark.sql.vectorized.ColumnarBatch import org.apache.comet.CometArrowAllocator /** * A reader that consumes Arrow data from an input channel, and produces Comet batches. */ case class StreamReader(channel: ReadableByteChannel, source: String) extends AutoCloseable { private val channelReader = new MessageChannelReader(new ReadChannel(channel), CometArrowAllocator) private var arrowReader = new ArrowStreamReader(channelReader, CometArrowAllocator) private var root = arrowReader.getVectorSchemaRoot def nextBatch(): Option[ColumnarBatch] = { if (arrowReader.loadNextBatch()) { Some(rootAsBatch(root)) } else { None } } private def rootAsBatch(root: VectorSchemaRoot): ColumnarBatch = { NativeUtil.rootAsBatch(root, arrowReader) } override def close(): Unit = { if (root != null) { arrowReader.close() root.close() arrowReader = null root = null } } } ================================================ FILE: common/src/main/scala/org/apache/spark/sql/comet/CastOverflowException.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.spark.sql.comet import org.apache.spark.SparkArithmeticException import org.apache.spark.sql.errors.QueryExecutionErrors.toSQLConf import org.apache.spark.sql.internal.SQLConf class CastOverflowException(t: String, from: String, to: String) extends SparkArithmeticException( "CAST_OVERFLOW", Map( "value" -> t, "sourceType" -> s""""$from"""", "targetType" -> s""""$to"""", "ansiConfig" -> toSQLConf(SQLConf.ANSI_ENABLED.key)), Array.empty, "") {} ================================================ FILE: common/src/main/scala/org/apache/spark/sql/comet/execution/arrow/ArrowReaderIterator.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.spark.sql.comet.execution.arrow import java.nio.channels.ReadableByteChannel import org.apache.spark.sql.vectorized.ColumnarBatch import org.apache.comet.vector._ class ArrowReaderIterator(channel: ReadableByteChannel, source: String) extends Iterator[ColumnarBatch] { private val reader = StreamReader(channel, source) private var batch = nextBatch() private var currentBatch: ColumnarBatch = null private var isClosed: Boolean = false override def hasNext: Boolean = { if (isClosed) { return false } if (batch.isDefined) { return true } // Release the previous batch. // If it is not released, when closing the reader, arrow library will complain about // memory leak. if (currentBatch != null) { currentBatch.close() currentBatch = null } batch = nextBatch() if (batch.isEmpty) { close() return false } true } override def next(): ColumnarBatch = { if (!hasNext) { throw new NoSuchElementException } val nextBatch = batch.get currentBatch = nextBatch batch = None currentBatch } private def nextBatch(): Option[ColumnarBatch] = { reader.nextBatch() } def close(): Unit = synchronized { if (!isClosed) { if (currentBatch != null) { currentBatch.close() currentBatch = null } reader.close() isClosed = true } } } ================================================ FILE: common/src/main/scala/org/apache/spark/sql/comet/execution/arrow/ArrowWriters.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.spark.sql.comet.execution.arrow import scala.jdk.CollectionConverters._ import org.apache.arrow.vector._ import org.apache.arrow.vector.complex._ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.SpecializedGetters import org.apache.spark.sql.comet.util.Utils import org.apache.spark.sql.errors.QueryExecutionErrors import org.apache.spark.sql.types._ import org.apache.spark.sql.vectorized.ColumnarArray /** * This file is mostly copied from Spark SQL's * org.apache.spark.sql.execution.arrow.ArrowWriter.scala. Comet shadows Arrow classes to avoid * potential conflicts with Spark's Arrow dependencies, hence we cannot reuse Spark's ArrowWriter * directly. * * Performance enhancement: https://github.com/apache/datafusion-comet/issues/888 */ private[arrow] object ArrowWriter { def create(root: VectorSchemaRoot): ArrowWriter = { val children = root.getFieldVectors().asScala.map { vector => vector.allocateNew() createFieldWriter(vector) } new ArrowWriter(root, children.toArray) } private[sql] def createFieldWriter(vector: ValueVector): ArrowFieldWriter = { val field = vector.getField() (Utils.fromArrowField(field), vector) match { case (BooleanType, vector: BitVector) => new BooleanWriter(vector) case (ByteType, vector: TinyIntVector) => new ByteWriter(vector) case (ShortType, vector: SmallIntVector) => new ShortWriter(vector) case (IntegerType, vector: IntVector) => new IntegerWriter(vector) case (LongType, vector: BigIntVector) => new LongWriter(vector) case (FloatType, vector: Float4Vector) => new FloatWriter(vector) case (DoubleType, vector: Float8Vector) => new DoubleWriter(vector) case (DecimalType.Fixed(precision, scale), vector: DecimalVector) => new DecimalWriter(vector, precision, scale) case (StringType, vector: VarCharVector) => new StringWriter(vector) case (StringType, vector: LargeVarCharVector) => new LargeStringWriter(vector) case (BinaryType, vector: VarBinaryVector) => new BinaryWriter(vector) case (BinaryType, vector: LargeVarBinaryVector) => new LargeBinaryWriter(vector) case (DateType, vector: DateDayVector) => new DateWriter(vector) case (TimestampType, vector: TimeStampMicroTZVector) => new TimestampWriter(vector) case (TimestampNTZType, vector: TimeStampMicroVector) => new TimestampNTZWriter(vector) case (ArrayType(_, _), vector: ListVector) => val elementVector = createFieldWriter(vector.getDataVector()) new ArrayWriter(vector, elementVector) case (MapType(_, _, _), vector: MapVector) => val structVector = vector.getDataVector.asInstanceOf[StructVector] val keyWriter = createFieldWriter(structVector.getChild(MapVector.KEY_NAME)) val valueWriter = createFieldWriter(structVector.getChild(MapVector.VALUE_NAME)) new MapWriter(vector, structVector, keyWriter, valueWriter) case (StructType(_), vector: StructVector) => val children = (0 until vector.size()).map { ordinal => createFieldWriter(vector.getChildByOrdinal(ordinal)) } new StructWriter(vector, children.toArray) case (NullType, vector: NullVector) => new NullWriter(vector) case (_: YearMonthIntervalType, vector: IntervalYearVector) => new IntervalYearWriter(vector) case (_: DayTimeIntervalType, vector: DurationVector) => new DurationWriter(vector) // case (CalendarIntervalType, vector: IntervalMonthDayNanoVector) => // new IntervalMonthDayNanoWriter(vector) case (dt, _) => throw QueryExecutionErrors.notSupportTypeError(dt) } } } class ArrowWriter(val root: VectorSchemaRoot, fields: Array[ArrowFieldWriter]) { def schema: StructType = Utils.fromArrowSchema(root.getSchema()) private var count: Int = 0 def write(row: InternalRow): Unit = { var i = 0 while (i < fields.length) { fields(i).write(row, i) i += 1 } count += 1 } def writeCol(input: ColumnarArray, columnIndex: Int): Unit = { fields(columnIndex).writeCol(input) count = input.numElements() } def writeColNoNull(input: ColumnarArray, columnIndex: Int): Unit = { fields(columnIndex).writeColNoNull(input) count = input.numElements() } def finish(): Unit = { root.setRowCount(count) fields.foreach(_.finish()) } def reset(): Unit = { root.setRowCount(0) count = 0 fields.foreach(_.reset()) } } private[arrow] abstract class ArrowFieldWriter { def valueVector: ValueVector def name: String = valueVector.getField().getName() def dataType: DataType = Utils.fromArrowField(valueVector.getField()) def nullable: Boolean = valueVector.getField().isNullable() def setNull(): Unit def setValue(input: SpecializedGetters, ordinal: Int): Unit private[arrow] var count: Int = 0 def write(input: SpecializedGetters, ordinal: Int): Unit = { if (input.isNullAt(ordinal)) { setNull() } else { setValue(input, ordinal) } count += 1 } def writeCol(input: ColumnarArray): Unit = { val inputNumElements = input.numElements() valueVector.setInitialCapacity(inputNumElements) while (count < inputNumElements) { if (input.isNullAt(count)) { setNull() } else { setValue(input, count) } count += 1 } } def writeColNoNull(input: ColumnarArray): Unit = { val inputNumElements = input.numElements() valueVector.setInitialCapacity(inputNumElements) while (count < inputNumElements) { setValue(input, count) count += 1 } } def finish(): Unit = { valueVector.setValueCount(count) } def reset(): Unit = { valueVector.reset() count = 0 } } private[arrow] class BooleanWriter(val valueVector: BitVector) extends ArrowFieldWriter { override def setNull(): Unit = { valueVector.setNull(count) } override def setValue(input: SpecializedGetters, ordinal: Int): Unit = { valueVector.setSafe(count, if (input.getBoolean(ordinal)) 1 else 0) } } private[arrow] class ByteWriter(val valueVector: TinyIntVector) extends ArrowFieldWriter { override def setNull(): Unit = { valueVector.setNull(count) } override def setValue(input: SpecializedGetters, ordinal: Int): Unit = { valueVector.setSafe(count, input.getByte(ordinal)) } } private[arrow] class ShortWriter(val valueVector: SmallIntVector) extends ArrowFieldWriter { override def setNull(): Unit = { valueVector.setNull(count) } override def setValue(input: SpecializedGetters, ordinal: Int): Unit = { valueVector.setSafe(count, input.getShort(ordinal)) } } private[arrow] class IntegerWriter(val valueVector: IntVector) extends ArrowFieldWriter { override def setNull(): Unit = { valueVector.setNull(count) } override def setValue(input: SpecializedGetters, ordinal: Int): Unit = { valueVector.setSafe(count, input.getInt(ordinal)) } } private[arrow] class LongWriter(val valueVector: BigIntVector) extends ArrowFieldWriter { override def setNull(): Unit = { valueVector.setNull(count) } override def setValue(input: SpecializedGetters, ordinal: Int): Unit = { valueVector.setSafe(count, input.getLong(ordinal)) } } private[arrow] class FloatWriter(val valueVector: Float4Vector) extends ArrowFieldWriter { override def setNull(): Unit = { valueVector.setNull(count) } override def setValue(input: SpecializedGetters, ordinal: Int): Unit = { valueVector.setSafe(count, input.getFloat(ordinal)) } } private[arrow] class DoubleWriter(val valueVector: Float8Vector) extends ArrowFieldWriter { override def setNull(): Unit = { valueVector.setNull(count) } override def setValue(input: SpecializedGetters, ordinal: Int): Unit = { valueVector.setSafe(count, input.getDouble(ordinal)) } } private[arrow] class DecimalWriter(val valueVector: DecimalVector, precision: Int, scale: Int) extends ArrowFieldWriter { override def setNull(): Unit = { valueVector.setNull(count) } override def setValue(input: SpecializedGetters, ordinal: Int): Unit = { val decimal = input.getDecimal(ordinal, precision, scale) if (decimal.changePrecision(precision, scale)) { valueVector.setSafe(count, decimal.toJavaBigDecimal) } else { setNull() } } } private[arrow] class StringWriter(val valueVector: VarCharVector) extends ArrowFieldWriter { override def setNull(): Unit = { valueVector.setNull(count) } override def setValue(input: SpecializedGetters, ordinal: Int): Unit = { val utf8 = input.getUTF8String(ordinal) val utf8ByteBuffer = utf8.getByteBuffer // todo: for off-heap UTF8String, how to pass in to arrow without copy? valueVector.setSafe(count, utf8ByteBuffer, utf8ByteBuffer.position(), utf8.numBytes()) } } private[arrow] class LargeStringWriter(val valueVector: LargeVarCharVector) extends ArrowFieldWriter { override def setNull(): Unit = { valueVector.setNull(count) } override def setValue(input: SpecializedGetters, ordinal: Int): Unit = { val utf8 = input.getUTF8String(ordinal) val utf8ByteBuffer = utf8.getByteBuffer // todo: for off-heap UTF8String, how to pass in to arrow without copy? valueVector.setSafe(count, utf8ByteBuffer, utf8ByteBuffer.position(), utf8.numBytes()) } } private[arrow] class BinaryWriter(val valueVector: VarBinaryVector) extends ArrowFieldWriter { override def setNull(): Unit = { valueVector.setNull(count) } override def setValue(input: SpecializedGetters, ordinal: Int): Unit = { val bytes = input.getBinary(ordinal) valueVector.setSafe(count, bytes, 0, bytes.length) } } private[arrow] class LargeBinaryWriter(val valueVector: LargeVarBinaryVector) extends ArrowFieldWriter { override def setNull(): Unit = { valueVector.setNull(count) } override def setValue(input: SpecializedGetters, ordinal: Int): Unit = { val bytes = input.getBinary(ordinal) valueVector.setSafe(count, bytes, 0, bytes.length) } } private[arrow] class DateWriter(val valueVector: DateDayVector) extends ArrowFieldWriter { override def setNull(): Unit = { valueVector.setNull(count) } override def setValue(input: SpecializedGetters, ordinal: Int): Unit = { valueVector.setSafe(count, input.getInt(ordinal)) } } private[arrow] class TimestampWriter(val valueVector: TimeStampMicroTZVector) extends ArrowFieldWriter { override def setNull(): Unit = { valueVector.setNull(count) } override def setValue(input: SpecializedGetters, ordinal: Int): Unit = { valueVector.setSafe(count, input.getLong(ordinal)) } } private[arrow] class TimestampNTZWriter(val valueVector: TimeStampMicroVector) extends ArrowFieldWriter { override def setNull(): Unit = { valueVector.setNull(count) } override def setValue(input: SpecializedGetters, ordinal: Int): Unit = { valueVector.setSafe(count, input.getLong(ordinal)) } } private[arrow] class ArrayWriter(val valueVector: ListVector, val elementWriter: ArrowFieldWriter) extends ArrowFieldWriter { override def setNull(): Unit = {} override def setValue(input: SpecializedGetters, ordinal: Int): Unit = { val array = input.getArray(ordinal) var i = 0 valueVector.startNewValue(count) while (i < array.numElements()) { elementWriter.write(array, i) i += 1 } valueVector.endValue(count, array.numElements()) } override def finish(): Unit = { super.finish() elementWriter.finish() } override def reset(): Unit = { super.reset() elementWriter.reset() } } private[arrow] class StructWriter( val valueVector: StructVector, children: Array[ArrowFieldWriter]) extends ArrowFieldWriter { override def setNull(): Unit = { var i = 0 while (i < children.length) { children(i).setNull() children(i).count += 1 i += 1 } valueVector.setNull(count) } override def setValue(input: SpecializedGetters, ordinal: Int): Unit = { val struct = input.getStruct(ordinal, children.length) var i = 0 valueVector.setIndexDefined(count) while (i < struct.numFields) { children(i).write(struct, i) i += 1 } } override def finish(): Unit = { super.finish() children.foreach(_.finish()) } override def reset(): Unit = { super.reset() children.foreach(_.reset()) } } private[arrow] class MapWriter( val valueVector: MapVector, val structVector: StructVector, val keyWriter: ArrowFieldWriter, val valueWriter: ArrowFieldWriter) extends ArrowFieldWriter { override def setNull(): Unit = {} override def setValue(input: SpecializedGetters, ordinal: Int): Unit = { val map = input.getMap(ordinal) valueVector.startNewValue(count) val keys = map.keyArray() val values = map.valueArray() var i = 0 while (i < map.numElements()) { structVector.setIndexDefined(keyWriter.count) keyWriter.write(keys, i) valueWriter.write(values, i) i += 1 } valueVector.endValue(count, map.numElements()) } override def finish(): Unit = { super.finish() keyWriter.finish() valueWriter.finish() } override def reset(): Unit = { super.reset() keyWriter.reset() valueWriter.reset() } } private[arrow] class NullWriter(val valueVector: NullVector) extends ArrowFieldWriter { override def setNull(): Unit = {} override def setValue(input: SpecializedGetters, ordinal: Int): Unit = {} } private[arrow] class IntervalYearWriter(val valueVector: IntervalYearVector) extends ArrowFieldWriter { override def setNull(): Unit = { valueVector.setNull(count) } override def setValue(input: SpecializedGetters, ordinal: Int): Unit = { valueVector.setSafe(count, input.getInt(ordinal)); } } private[arrow] class DurationWriter(val valueVector: DurationVector) extends ArrowFieldWriter { override def setNull(): Unit = { valueVector.setNull(count) } override def setValue(input: SpecializedGetters, ordinal: Int): Unit = { valueVector.setSafe(count, input.getLong(ordinal)) } } private[arrow] class IntervalMonthDayNanoWriter(val valueVector: IntervalMonthDayNanoVector) extends ArrowFieldWriter { override def setNull(): Unit = { valueVector.setNull(count) } override def setValue(input: SpecializedGetters, ordinal: Int): Unit = { val ci = input.getInterval(ordinal) valueVector.setSafe(count, ci.months, ci.days, Math.multiplyExact(ci.microseconds, 1000L)) } } ================================================ FILE: common/src/main/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowConverters.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.spark.sql.comet.execution.arrow import org.apache.arrow.memory.BufferAllocator import org.apache.arrow.vector.VectorSchemaRoot import org.apache.arrow.vector.types.pojo.Schema import org.apache.spark.TaskContext import org.apache.spark.internal.Logging import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.comet.util.Utils import org.apache.spark.sql.types.StructType import org.apache.spark.sql.vectorized.{ColumnarArray, ColumnarBatch} import org.apache.comet.CometArrowAllocator import org.apache.comet.vector.NativeUtil object CometArrowConverters extends Logging { // This is similar how Spark converts internal row to Arrow format except that it is transforming // the result batch to Comet's ColumnarBatch instead of serialized bytes. // There's another big difference that Comet may consume the ColumnarBatch by exporting it to // the native side. Hence, we need to: // 1. reset the Arrow writer after the ColumnarBatch is consumed // 2. close the allocator when the task is finished but not when the iterator is all consumed // The reason for the second point is that when ColumnarBatch is exported to the native side, the // exported process increases the reference count of the Arrow vectors. The reference count is // only decreased when the native plan is done with the vectors, which is usually longer than // all the ColumnarBatches are consumed. abstract private[sql] class ArrowBatchIterBase( schema: StructType, timeZoneId: String, context: TaskContext) extends Iterator[ColumnarBatch] with AutoCloseable { protected val arrowSchema: Schema = Utils.toArrowSchema(schema, timeZoneId) // Reuse the same root allocator here. protected val allocator: BufferAllocator = CometArrowAllocator.newChildAllocator(s"to${this.getClass.getSimpleName}", 0, Long.MaxValue) protected val root: VectorSchemaRoot = VectorSchemaRoot.create(arrowSchema, allocator) protected val arrowWriter: ArrowWriter = ArrowWriter.create(root) protected var currentBatch: ColumnarBatch = null protected var closed: Boolean = false Option(context).foreach { _.addTaskCompletionListener[Unit] { _ => close(true) } } override def close(): Unit = { close(false) } protected def close(closeAllocator: Boolean): Unit = { try { if (!closed) { if (currentBatch != null) { arrowWriter.reset() currentBatch.close() currentBatch = null } root.close() closed = true } } finally { // the allocator shall be closed when the task is finished if (closeAllocator) { allocator.close() } } } override def next(): ColumnarBatch = { currentBatch = nextBatch() currentBatch } protected def nextBatch(): ColumnarBatch } private[sql] class RowToArrowBatchIter( rowIter: Iterator[InternalRow], schema: StructType, maxRecordsPerBatch: Long, timeZoneId: String, context: TaskContext) extends ArrowBatchIterBase(schema, timeZoneId, context) with AutoCloseable { override def hasNext: Boolean = rowIter.hasNext || { close(false) false } override protected def nextBatch(): ColumnarBatch = { if (rowIter.hasNext) { // the arrow writer shall be reset before writing the next batch arrowWriter.reset() var rowCount = 0L while (rowIter.hasNext && (maxRecordsPerBatch <= 0 || rowCount < maxRecordsPerBatch)) { val row = rowIter.next() arrowWriter.write(row) rowCount += 1 } arrowWriter.finish() NativeUtil.rootAsBatch(root) } else { null } } } def rowToArrowBatchIter( rowIter: Iterator[InternalRow], schema: StructType, maxRecordsPerBatch: Long, timeZoneId: String, context: TaskContext): Iterator[ColumnarBatch] = { new RowToArrowBatchIter(rowIter, schema, maxRecordsPerBatch, timeZoneId, context) } private[sql] class ColumnBatchToArrowBatchIter( colBatch: ColumnarBatch, schema: StructType, maxRecordsPerBatch: Int, timeZoneId: String, context: TaskContext) extends ArrowBatchIterBase(schema, timeZoneId, context) with AutoCloseable { private var rowsProduced: Int = 0 override def hasNext: Boolean = rowsProduced < colBatch.numRows() || { close(false) false } override protected def nextBatch(): ColumnarBatch = { val rowsInBatch = colBatch.numRows() if (rowsProduced < rowsInBatch) { // the arrow writer shall be reset before writing the next batch arrowWriter.reset() val rowsToProduce = if (maxRecordsPerBatch <= 0) rowsInBatch - rowsProduced else Math.min(maxRecordsPerBatch, rowsInBatch - rowsProduced) for (columnIndex <- 0 until colBatch.numCols()) { val column = colBatch.column(columnIndex) val columnArray = new ColumnarArray(column, rowsProduced, rowsToProduce) if (column.hasNull) { arrowWriter.writeCol(columnArray, columnIndex) } else { arrowWriter.writeColNoNull(columnArray, columnIndex) } } rowsProduced += rowsToProduce arrowWriter.finish() NativeUtil.rootAsBatch(root) } else { null } } } def columnarBatchToArrowBatchIter( colBatch: ColumnarBatch, schema: StructType, maxRecordsPerBatch: Int, timeZoneId: String, context: TaskContext): Iterator[ColumnarBatch] = { new ColumnBatchToArrowBatchIter(colBatch, schema, maxRecordsPerBatch, timeZoneId, context) } } ================================================ FILE: common/src/main/scala/org/apache/spark/sql/comet/parquet/CometParquetReadSupport.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.spark.sql.comet.parquet import java.util.{Locale, UUID} import scala.jdk.CollectionConverters._ import org.apache.parquet.schema._ import org.apache.parquet.schema.LogicalTypeAnnotation.ListLogicalTypeAnnotation import org.apache.parquet.schema.Type.Repetition import org.apache.spark.sql.errors.QueryExecutionErrors import org.apache.spark.sql.execution.datasources.parquet.ParquetUtils import org.apache.spark.sql.types._ /** * This class is copied & slightly modified from [[ParquetReadSupport]] in Spark. Changes: * - This doesn't extend from Parquet's `ReadSupport` class since that is used for row-based * Parquet reader. Therefore, there is no `init`, `prepareForRead` as well as other methods * that are unused. */ object CometParquetReadSupport { val SPARK_PARQUET_SCHEMA_NAME = "spark_schema" val EMPTY_MESSAGE: MessageType = Types.buildMessage().named(SPARK_PARQUET_SCHEMA_NAME) def generateFakeColumnName: String = s"_fake_name_${UUID.randomUUID()}" def clipParquetSchema( parquetSchema: MessageType, catalystSchema: StructType, caseSensitive: Boolean, useFieldId: Boolean, ignoreMissingIds: Boolean): MessageType = { if (!ignoreMissingIds && !containsFieldIds(parquetSchema) && ParquetUtils.hasFieldIds(catalystSchema)) { throw new RuntimeException( "Spark read schema expects field Ids, " + "but Parquet file schema doesn't contain any field Ids.\n" + "Please remove the field ids from Spark schema or ignore missing ids by " + "setting `spark.sql.parquet.fieldId.read.ignoreMissing = true`\n" + s""" |Spark read schema: |${catalystSchema.prettyJson} | |Parquet file schema: |${parquetSchema.toString} |""".stripMargin) } clipParquetSchema(parquetSchema, catalystSchema, caseSensitive, useFieldId) } /** * Tailors `parquetSchema` according to `catalystSchema` by removing column paths don't exist in * `catalystSchema`, and adding those only exist in `catalystSchema`. */ def clipParquetSchema( parquetSchema: MessageType, catalystSchema: StructType, caseSensitive: Boolean, useFieldId: Boolean): MessageType = { val clippedParquetFields = clipParquetGroupFields( parquetSchema.asGroupType(), catalystSchema, caseSensitive, useFieldId) if (clippedParquetFields.isEmpty) { EMPTY_MESSAGE } else { Types .buildMessage() .addFields(clippedParquetFields: _*) .named(SPARK_PARQUET_SCHEMA_NAME) } } private def clipParquetType( parquetType: Type, catalystType: DataType, caseSensitive: Boolean, useFieldId: Boolean): Type = { val newParquetType = catalystType match { case t: ArrayType if !isPrimitiveCatalystType(t.elementType) => // Only clips array types with nested type as element type. clipParquetListType(parquetType.asGroupType(), t.elementType, caseSensitive, useFieldId) case t: MapType if !isPrimitiveCatalystType(t.keyType) || !isPrimitiveCatalystType(t.valueType) => // Only clips map types with nested key type or value type clipParquetMapType( parquetType.asGroupType(), t.keyType, t.valueType, caseSensitive, useFieldId) case t: StructType => clipParquetGroup(parquetType.asGroupType(), t, caseSensitive, useFieldId) case _ => // UDTs and primitive types are not clipped. For UDTs, a clipped version might not be able // to be mapped to desired user-space types. So UDTs shouldn't participate schema merging. parquetType } if (useFieldId && parquetType.getId != null) { newParquetType.withId(parquetType.getId.intValue()) } else { newParquetType } } /** * Whether a Catalyst [[DataType]] is primitive. Primitive [[DataType]] is not equivalent to * [[AtomicType]]. For example, [[CalendarIntervalType]] is primitive, but it's not an * [[AtomicType]]. */ private def isPrimitiveCatalystType(dataType: DataType): Boolean = { dataType match { case _: ArrayType | _: MapType | _: StructType => false case _ => true } } /** * Clips a Parquet [[GroupType]] which corresponds to a Catalyst [[ArrayType]]. The element type * of the [[ArrayType]] should also be a nested type, namely an [[ArrayType]], a [[MapType]], or * a [[StructType]]. */ private def clipParquetListType( parquetList: GroupType, elementType: DataType, caseSensitive: Boolean, useFieldId: Boolean): Type = { // Precondition of this method, should only be called for lists with nested element types. assert(!isPrimitiveCatalystType(elementType)) // Unannotated repeated group should be interpreted as required list of required element, so // list element type is just the group itself. Clip it. if (parquetList.getLogicalTypeAnnotation == null && parquetList.isRepetition(Repetition.REPEATED)) { clipParquetType(parquetList, elementType, caseSensitive, useFieldId) } else { assert( parquetList.getLogicalTypeAnnotation.isInstanceOf[ListLogicalTypeAnnotation], "Invalid Parquet schema. " + "Logical type annotation of annotated Parquet lists must be ListLogicalTypeAnnotation: " + parquetList.toString) assert( parquetList.getFieldCount == 1 && parquetList .getType(0) .isRepetition(Repetition.REPEATED), "Invalid Parquet schema. " + "LIST-annotated group should only have exactly one repeated field: " + parquetList) // Precondition of this method, should only be called for lists with nested element types. assert(!parquetList.getType(0).isPrimitive) val repeatedGroup = parquetList.getType(0).asGroupType() // If the repeated field is a group with multiple fields, or the repeated field is a group // with one field and is named either "array" or uses the LIST-annotated group's name with // "_tuple" appended then the repeated type is the element type and elements are required. // Build a new LIST-annotated group with clipped `repeatedGroup` as element type and the // only field. if (repeatedGroup.getFieldCount > 1 || repeatedGroup.getName == "array" || repeatedGroup.getName == parquetList.getName + "_tuple") { Types .buildGroup(parquetList.getRepetition) .as(LogicalTypeAnnotation.listType()) .addField(clipParquetType(repeatedGroup, elementType, caseSensitive, useFieldId)) .named(parquetList.getName) } else { val newRepeatedGroup = Types .repeatedGroup() .addField( clipParquetType(repeatedGroup.getType(0), elementType, caseSensitive, useFieldId)) .named(repeatedGroup.getName) val newElementType = if (useFieldId && repeatedGroup.getId != null) { newRepeatedGroup.withId(repeatedGroup.getId.intValue()) } else { newRepeatedGroup } // Otherwise, the repeated field's type is the element type with the repeated field's // repetition. Types .buildGroup(parquetList.getRepetition) .as(LogicalTypeAnnotation.listType()) .addField(newElementType) .named(parquetList.getName) } } } /** * Clips a Parquet [[GroupType]] which corresponds to a Catalyst [[MapType]]. Either key type or * value type of the [[MapType]] must be a nested type, namely an [[ArrayType]], a [[MapType]], * or a [[StructType]]. */ private def clipParquetMapType( parquetMap: GroupType, keyType: DataType, valueType: DataType, caseSensitive: Boolean, useFieldId: Boolean): GroupType = { // Precondition of this method, only handles maps with nested key types or value types. assert(!isPrimitiveCatalystType(keyType) || !isPrimitiveCatalystType(valueType)) val repeatedGroup = parquetMap.getType(0).asGroupType() val parquetKeyType = repeatedGroup.getType(0) val parquetValueType = repeatedGroup.getType(1) val clippedRepeatedGroup = { val newRepeatedGroup = Types .repeatedGroup() .as(repeatedGroup.getLogicalTypeAnnotation) .addField(clipParquetType(parquetKeyType, keyType, caseSensitive, useFieldId)) .addField(clipParquetType(parquetValueType, valueType, caseSensitive, useFieldId)) .named(repeatedGroup.getName) if (useFieldId && repeatedGroup.getId != null) { newRepeatedGroup.withId(repeatedGroup.getId.intValue()) } else { newRepeatedGroup } } Types .buildGroup(parquetMap.getRepetition) .as(parquetMap.getLogicalTypeAnnotation) .addField(clippedRepeatedGroup) .named(parquetMap.getName) } /** * Clips a Parquet [[GroupType]] which corresponds to a Catalyst [[StructType]]. * * @return * A clipped [[GroupType]], which has at least one field. * @note * Parquet doesn't allow creating empty [[GroupType]] instances except for empty * [[MessageType]]. Because it's legal to construct an empty requested schema for column * pruning. */ private def clipParquetGroup( parquetRecord: GroupType, structType: StructType, caseSensitive: Boolean, useFieldId: Boolean): GroupType = { val clippedParquetFields = clipParquetGroupFields(parquetRecord, structType, caseSensitive, useFieldId) Types .buildGroup(parquetRecord.getRepetition) .as(parquetRecord.getLogicalTypeAnnotation) .addFields(clippedParquetFields: _*) .named(parquetRecord.getName) } /** * Clips a Parquet [[GroupType]] which corresponds to a Catalyst [[StructType]]. * * @return * A list of clipped [[GroupType]] fields, which can be empty. */ private def clipParquetGroupFields( parquetRecord: GroupType, structType: StructType, caseSensitive: Boolean, useFieldId: Boolean): Seq[Type] = { val toParquet = new CometSparkToParquetSchemaConverter( writeLegacyParquetFormat = false, useFieldId = useFieldId) lazy val caseSensitiveParquetFieldMap = parquetRecord.getFields.asScala.map(f => f.getName -> f).toMap lazy val caseInsensitiveParquetFieldMap = parquetRecord.getFields.asScala.groupBy(_.getName.toLowerCase(Locale.ROOT)) lazy val idToParquetFieldMap = parquetRecord.getFields.asScala.filter(_.getId != null).groupBy(f => f.getId.intValue()) def matchCaseSensitiveField(f: StructField): Type = { caseSensitiveParquetFieldMap .get(f.name) .map(clipParquetType(_, f.dataType, caseSensitive, useFieldId)) .getOrElse(toParquet.convertField(f)) } def matchCaseInsensitiveField(f: StructField): Type = { // Do case-insensitive resolution only if in case-insensitive mode caseInsensitiveParquetFieldMap .get(f.name.toLowerCase(Locale.ROOT)) .map { parquetTypes => if (parquetTypes.size > 1) { // Need to fail if there is ambiguity, i.e. more than one field is matched val parquetTypesString = parquetTypes.map(_.getName).mkString("[", ", ", "]") throw QueryExecutionErrors.foundDuplicateFieldInCaseInsensitiveModeError( f.name, parquetTypesString) } else { clipParquetType(parquetTypes.head, f.dataType, caseSensitive, useFieldId) } } .getOrElse(toParquet.convertField(f)) } def matchIdField(f: StructField): Type = { val fieldId = ParquetUtils.getFieldId(f) idToParquetFieldMap .get(fieldId) .map { parquetTypes => if (parquetTypes.size > 1) { // Need to fail if there is ambiguity, i.e. more than one field is matched val parquetTypesString = parquetTypes.map(_.getName).mkString("[", ", ", "]") throw QueryExecutionErrors.foundDuplicateFieldInFieldIdLookupModeError( fieldId, parquetTypesString) } else { clipParquetType(parquetTypes.head, f.dataType, caseSensitive, useFieldId) } } .getOrElse { // When there is no ID match, we use a fake name to avoid a name match by accident // We need this name to be unique as well, otherwise there will be type conflicts toParquet.convertField(f.copy(name = generateFakeColumnName)) } } val shouldMatchById = useFieldId && ParquetUtils.hasFieldIds(structType) structType.map { f => if (shouldMatchById && ParquetUtils.hasFieldId(f)) { matchIdField(f) } else if (caseSensitive) { matchCaseSensitiveField(f) } else { matchCaseInsensitiveField(f) } } } /** * Whether the parquet schema contains any field IDs. */ private def containsFieldIds(schema: Type): Boolean = schema match { case p: PrimitiveType => p.getId != null // We don't require all fields to have IDs, so we use `exists` here. case g: GroupType => g.getId != null || g.getFields.asScala.exists(containsFieldIds) } } ================================================ FILE: common/src/main/scala/org/apache/spark/sql/comet/parquet/CometSparkToParquetSchemaConverter.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.spark.sql.comet.parquet import org.apache.hadoop.conf.Configuration import org.apache.parquet.schema._ import org.apache.parquet.schema.LogicalTypeAnnotation._ import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName._ import org.apache.parquet.schema.Type.Repetition._ import org.apache.spark.sql.errors.QueryCompilationErrors import org.apache.spark.sql.execution.datasources.parquet.ParquetSchemaConverter import org.apache.spark.sql.execution.datasources.parquet.ParquetUtils import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ import org.apache.comet.parquet.CometParquetUtils /** * This class is copied & modified from Spark's [[SparkToParquetSchemaConverter]] class. */ class CometSparkToParquetSchemaConverter( writeLegacyParquetFormat: Boolean = SQLConf.PARQUET_WRITE_LEGACY_FORMAT.defaultValue.get, outputTimestampType: SQLConf.ParquetOutputTimestampType.Value = SQLConf.ParquetOutputTimestampType.INT96, useFieldId: Boolean = CometParquetUtils.writeFieldId(new SQLConf)) { def this(conf: SQLConf) = this( writeLegacyParquetFormat = conf.writeLegacyParquetFormat, outputTimestampType = conf.parquetOutputTimestampType, useFieldId = CometParquetUtils.writeFieldId(conf)) def this(conf: Configuration) = this( writeLegacyParquetFormat = conf.get(SQLConf.PARQUET_WRITE_LEGACY_FORMAT.key).toBoolean, outputTimestampType = SQLConf.ParquetOutputTimestampType.withName( conf.get(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key)), useFieldId = CometParquetUtils.writeFieldId(conf)) /** * Converts a Spark SQL [[StructType]] to a Parquet [[MessageType]]. */ def convert(catalystSchema: StructType): MessageType = { Types .buildMessage() .addFields(catalystSchema.map(convertField): _*) .named(ParquetSchemaConverter.SPARK_PARQUET_SCHEMA_NAME) } /** * Converts a Spark SQL [[StructField]] to a Parquet [[Type]]. */ def convertField(field: StructField): Type = { val converted = convertField(field, if (field.nullable) OPTIONAL else REQUIRED) if (useFieldId && ParquetUtils.hasFieldId(field)) { converted.withId(ParquetUtils.getFieldId(field)) } else { converted } } private def convertField(field: StructField, repetition: Type.Repetition): Type = { field.dataType match { // =================== // Simple atomic types // =================== case BooleanType => Types.primitive(BOOLEAN, repetition).named(field.name) case ByteType => Types .primitive(INT32, repetition) .as(LogicalTypeAnnotation.intType(8, true)) .named(field.name) case ShortType => Types .primitive(INT32, repetition) .as(LogicalTypeAnnotation.intType(16, true)) .named(field.name) case IntegerType => Types.primitive(INT32, repetition).named(field.name) case LongType => Types.primitive(INT64, repetition).named(field.name) case FloatType => Types.primitive(FLOAT, repetition).named(field.name) case DoubleType => Types.primitive(DOUBLE, repetition).named(field.name) case StringType => Types .primitive(BINARY, repetition) .as(LogicalTypeAnnotation.stringType()) .named(field.name) case DateType => Types .primitive(INT32, repetition) .as(LogicalTypeAnnotation.dateType()) .named(field.name) // NOTE: Spark SQL can write timestamp values to Parquet using INT96, TIMESTAMP_MICROS or // TIMESTAMP_MILLIS. TIMESTAMP_MICROS is recommended but INT96 is the default to keep the // behavior same as before. // // As stated in PARQUET-323, Parquet `INT96` was originally introduced to represent nanosecond // timestamp in Impala for some historical reasons. It's not recommended to be used for any // other types and will probably be deprecated in some future version of parquet-format spec. // That's the reason why parquet-format spec only defines `TIMESTAMP_MILLIS` and // `TIMESTAMP_MICROS` which are both logical types annotating `INT64`. // // Originally, Spark SQL uses the same nanosecond timestamp type as Impala and Hive. Starting // from Spark 1.5.0, we resort to a timestamp type with microsecond precision so that we can // store a timestamp into a `Long`. This design decision is subject to change though, for // example, we may resort to nanosecond precision in the future. case TimestampType => outputTimestampType match { case SQLConf.ParquetOutputTimestampType.INT96 => Types.primitive(INT96, repetition).named(field.name) case SQLConf.ParquetOutputTimestampType.TIMESTAMP_MICROS => Types .primitive(INT64, repetition) .as(LogicalTypeAnnotation.timestampType(true, TimeUnit.MICROS)) .named(field.name) case SQLConf.ParquetOutputTimestampType.TIMESTAMP_MILLIS => Types .primitive(INT64, repetition) .as(LogicalTypeAnnotation.timestampType(true, TimeUnit.MILLIS)) .named(field.name) } case TimestampNTZType => Types .primitive(INT64, repetition) .as(LogicalTypeAnnotation.timestampType(false, TimeUnit.MICROS)) .named(field.name) case BinaryType => Types.primitive(BINARY, repetition).named(field.name) // ====================== // Decimals (legacy mode) // ====================== // Spark 1.4.x and prior versions only support decimals with a maximum precision of 18 and // always store decimals in fixed-length byte arrays. To keep compatibility with these older // versions, here we convert decimals with all precisions to `FIXED_LEN_BYTE_ARRAY` annotated // by `DECIMAL`. case DecimalType.Fixed(precision, scale) if writeLegacyParquetFormat => Types .primitive(FIXED_LEN_BYTE_ARRAY, repetition) .as(LogicalTypeAnnotation.decimalType(scale, precision)) .length(Decimal.minBytesForPrecision(precision)) .named(field.name) // ======================== // Decimals (standard mode) // ======================== // Uses INT32 for 1 <= precision <= 9 case DecimalType.Fixed(precision, scale) if precision <= Decimal.MAX_INT_DIGITS && !writeLegacyParquetFormat => Types .primitive(INT32, repetition) .as(LogicalTypeAnnotation.decimalType(scale, precision)) .named(field.name) // Uses INT64 for 1 <= precision <= 18 case DecimalType.Fixed(precision, scale) if precision <= Decimal.MAX_LONG_DIGITS && !writeLegacyParquetFormat => Types .primitive(INT64, repetition) .as(LogicalTypeAnnotation.decimalType(scale, precision)) .named(field.name) // Uses FIXED_LEN_BYTE_ARRAY for all other precisions case DecimalType.Fixed(precision, scale) if !writeLegacyParquetFormat => Types .primitive(FIXED_LEN_BYTE_ARRAY, repetition) .as(LogicalTypeAnnotation.decimalType(scale, precision)) .length(Decimal.minBytesForPrecision(precision)) .named(field.name) // =================================== // ArrayType and MapType (legacy mode) // =================================== // Spark 1.4.x and prior versions convert `ArrayType` with nullable elements into a 3-level // `LIST` structure. This behavior is somewhat a hybrid of parquet-hive and parquet-avro // (1.6.0rc3): the 3-level structure is similar to parquet-hive while the 3rd level element // field name "array" is borrowed from parquet-avro. case ArrayType(elementType, nullable @ true) if writeLegacyParquetFormat => // group (LIST) { // optional group bag { // repeated array; // } // } // This should not use `listOfElements` here because this new method checks if the // element name is `element` in the `GroupType` and throws an exception if not. // As mentioned above, Spark prior to 1.4.x writes `ArrayType` as `LIST` but with // `array` as its element name as below. Therefore, we build manually // the correct group type here via the builder. (See SPARK-16777) Types .buildGroup(repetition) .as(LogicalTypeAnnotation.listType()) .addField( Types .buildGroup(REPEATED) // "array" is the name chosen by parquet-hive (1.7.0 and prior version) .addField(convertField(StructField("array", elementType, nullable))) .named("bag")) .named(field.name) // Spark 1.4.x and prior versions convert ArrayType with non-nullable elements into a 2-level // LIST structure. This behavior mimics parquet-avro (1.6.0rc3). Note that this case is // covered by the backwards-compatibility rules implemented in `isElementType()`. case ArrayType(elementType, nullable @ false) if writeLegacyParquetFormat => // group (LIST) { // repeated element; // } // Here too, we should not use `listOfElements`. (See SPARK-16777) Types .buildGroup(repetition) .as(LogicalTypeAnnotation.listType()) // "array" is the name chosen by parquet-avro (1.7.0 and prior version) .addField(convertField(StructField("array", elementType, nullable), REPEATED)) .named(field.name) // Spark 1.4.x and prior versions convert MapType into a 3-level group annotated by // MAP_KEY_VALUE. This is covered by `convertGroupField(field: GroupType): DataType`. case MapType(keyType, valueType, valueContainsNull) if writeLegacyParquetFormat => // group (MAP) { // repeated group map (MAP_KEY_VALUE) { // required key; // value; // } // } ConversionPatterns.mapType( repetition, field.name, convertField(StructField("key", keyType, nullable = false)), convertField(StructField("value", valueType, valueContainsNull))) // ===================================== // ArrayType and MapType (standard mode) // ===================================== case ArrayType(elementType, containsNull) if !writeLegacyParquetFormat => // group (LIST) { // repeated group list { // element; // } // } Types .buildGroup(repetition) .as(LogicalTypeAnnotation.listType()) .addField( Types .repeatedGroup() .addField(convertField(StructField("element", elementType, containsNull))) .named("list")) .named(field.name) case MapType(keyType, valueType, valueContainsNull) => // group (MAP) { // repeated group key_value { // required key; // value; // } // } Types .buildGroup(repetition) .as(LogicalTypeAnnotation.mapType()) .addField( Types .repeatedGroup() .addField(convertField(StructField("key", keyType, nullable = false))) .addField(convertField(StructField("value", valueType, valueContainsNull))) .named("key_value")) .named(field.name) // =========== // Other types // =========== case StructType(fields) => fields .foldLeft(Types.buildGroup(repetition)) { (builder, field) => builder.addField(convertField(field)) } .named(field.name) case udt: UserDefinedType[_] => convertField(field.copy(dataType = udt.sqlType)) case _ => throw QueryCompilationErrors.cannotConvertDataTypeToParquetTypeError(field) } } } ================================================ FILE: common/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.spark.sql.comet.util import java.io.{DataInputStream, DataOutputStream, File} import java.nio.ByteBuffer import java.nio.channels.Channels import scala.jdk.CollectionConverters._ import org.apache.arrow.c.CDataDictionaryProvider import org.apache.arrow.vector._ import org.apache.arrow.vector.complex.{ListVector, MapVector, StructVector} import org.apache.arrow.vector.dictionary.DictionaryProvider import org.apache.arrow.vector.ipc.{ArrowStreamReader, ArrowStreamWriter} import org.apache.arrow.vector.types._ import org.apache.arrow.vector.types.pojo.{ArrowType, Field, FieldType, Schema} import org.apache.arrow.vector.util.VectorSchemaRootAppender import org.apache.spark.{SparkEnv, SparkException} import org.apache.spark.internal.Logging import org.apache.spark.io.CompressionCodec import org.apache.spark.sql.comet.execution.arrow.ArrowReaderIterator import org.apache.spark.sql.types._ import org.apache.spark.sql.vectorized.ColumnarBatch import org.apache.spark.util.io.{ChunkedByteBuffer, ChunkedByteBufferOutputStream} import org.apache.comet.Constants.COMET_CONF_DIR_ENV import org.apache.comet.shims.CometTypeShim import org.apache.comet.vector.CometVector object Utils extends CometTypeShim with Logging { def getConfPath(confFileName: String): String = { sys.env .get(COMET_CONF_DIR_ENV) .map { t => new File(s"$t${File.separator}$confFileName") } .filter(_.isFile) .map(_.getAbsolutePath) .orNull } def stringToSeq(str: String): Seq[String] = { str.split(",").map(_.trim()).filter(_.nonEmpty) } /** bridges the function call to Spark's Util */ def getSimpleName(cls: Class[_]): String = { org.apache.spark.util.Utils.getSimpleName(cls) } def fromArrowField(field: Field): DataType = { field.getType match { case _: ArrowType.Map => val elementField = field.getChildren.get(0) val keyType = fromArrowField(elementField.getChildren.get(0)) val valueType = fromArrowField(elementField.getChildren.get(1)) MapType(keyType, valueType, elementField.getChildren.get(1).isNullable) case ArrowType.List.INSTANCE => val elementField = field.getChildren().get(0) val elementType = fromArrowField(elementField) ArrayType(elementType, containsNull = elementField.isNullable) case ArrowType.Struct.INSTANCE => val fields = field.getChildren().asScala.map { child => val dt = fromArrowField(child) StructField(child.getName, dt, child.isNullable) } StructType(fields.toSeq) case arrowType => fromArrowType(arrowType) } } def fromArrowType(dt: ArrowType): DataType = dt match { case ArrowType.Bool.INSTANCE => BooleanType case int: ArrowType.Int if int.getIsSigned && int.getBitWidth == 8 => ByteType case int: ArrowType.Int if int.getIsSigned && int.getBitWidth == 8 * 2 => ShortType case int: ArrowType.Int if int.getIsSigned && int.getBitWidth == 8 * 4 => IntegerType case int: ArrowType.Int if int.getIsSigned && int.getBitWidth == 8 * 8 => LongType case float: ArrowType.FloatingPoint if float.getPrecision == FloatingPointPrecision.SINGLE => FloatType case float: ArrowType.FloatingPoint if float.getPrecision == FloatingPointPrecision.DOUBLE => DoubleType case ArrowType.Utf8.INSTANCE => StringType case ArrowType.Binary.INSTANCE => BinaryType case _: ArrowType.FixedSizeBinary => BinaryType case d: ArrowType.Decimal => DecimalType(d.getPrecision, d.getScale) case date: ArrowType.Date if date.getUnit == DateUnit.DAY => DateType case ts: ArrowType.Timestamp if ts.getUnit == TimeUnit.MICROSECOND && ts.getTimezone == null => TimestampNTZType case ts: ArrowType.Timestamp if ts.getUnit == TimeUnit.MICROSECOND => TimestampType case ArrowType.Null.INSTANCE => NullType case yi: ArrowType.Interval if yi.getUnit == IntervalUnit.YEAR_MONTH => YearMonthIntervalType() case di: ArrowType.Interval if di.getUnit == IntervalUnit.DAY_TIME => DayTimeIntervalType() case _ => throw new UnsupportedOperationException(s"Unsupported data type: ${dt.toString}") } def fromArrowSchema(schema: Schema): StructType = { StructType(schema.getFields.asScala.map { field => val dt = fromArrowField(field) StructField(field.getName, dt, field.isNullable) }.toArray) } /** Maps data type from Spark to Arrow. NOTE: timeZoneId required for TimestampTypes */ def toArrowType(dt: DataType, timeZoneId: String): ArrowType = dt match { case BooleanType => ArrowType.Bool.INSTANCE case ByteType => new ArrowType.Int(8, true) case ShortType => new ArrowType.Int(8 * 2, true) case IntegerType => new ArrowType.Int(8 * 4, true) case LongType => new ArrowType.Int(8 * 8, true) case FloatType => new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE) case DoubleType => new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE) case _: StringType => ArrowType.Utf8.INSTANCE case dt if isStringCollationType(dt) => ArrowType.Utf8.INSTANCE case BinaryType => ArrowType.Binary.INSTANCE case DecimalType.Fixed(precision, scale) => new ArrowType.Decimal(precision, scale, 128) case DateType => new ArrowType.Date(DateUnit.DAY) case TimestampType => if (timeZoneId == null) { throw new UnsupportedOperationException( s"${TimestampType.catalogString} must supply timeZoneId parameter") } else { new ArrowType.Timestamp(TimeUnit.MICROSECOND, timeZoneId) } case TimestampNTZType => new ArrowType.Timestamp(TimeUnit.MICROSECOND, null) case _ => throw new UnsupportedOperationException( s"Unsupported data type: [${dt.getClass.getName}] ${dt.catalogString}") } /** Maps field from Spark to Arrow. NOTE: timeZoneId required for TimestampType */ def toArrowField(name: String, dt: DataType, nullable: Boolean, timeZoneId: String): Field = { dt match { case ArrayType(elementType, containsNull) => val fieldType = new FieldType(nullable, ArrowType.List.INSTANCE, null) new Field( name, fieldType, Seq(toArrowField("element", elementType, containsNull, timeZoneId)).asJava) case StructType(fields) => val fieldType = new FieldType(nullable, ArrowType.Struct.INSTANCE, null) new Field( name, fieldType, fields .map { field => toArrowField(field.name, field.dataType, field.nullable, timeZoneId) } .toSeq .asJava) case MapType(keyType, valueType, valueContainsNull) => val mapType = new FieldType(nullable, new ArrowType.Map(false), null) // Note: Map Type struct can not be null, Struct Type key field can not be null new Field( name, mapType, Seq( toArrowField( MapVector.DATA_VECTOR_NAME, new StructType() .add(MapVector.KEY_NAME, keyType, nullable = false) .add(MapVector.VALUE_NAME, valueType, nullable = valueContainsNull), nullable = false, timeZoneId)).asJava) case dataType => val fieldType = new FieldType(nullable, toArrowType(dataType, timeZoneId), null) new Field(name, fieldType, Seq.empty[Field].asJava) } } /** * Maps schema from Spark to Arrow. NOTE: timeZoneId required for TimestampType in StructType */ def toArrowSchema(schema: StructType, timeZoneId: String): Schema = { new Schema(schema.map { field => toArrowField(field.name, field.dataType, field.nullable, timeZoneId) }.asJava) } /** * Serializes a list of `ColumnarBatch` into an output stream. This method must be in `spark` * package because `ChunkedByteBufferOutputStream` is spark private class. As it uses Arrow * classes, it must be in `common` module. * * @param batches * the output batches, each batch is a list of Arrow vectors wrapped in `CometVector` * @param out * the output stream */ def serializeBatches(batches: Iterator[ColumnarBatch]): Iterator[(Long, ChunkedByteBuffer)] = { batches.map { batch => val dictionaryProvider: CDataDictionaryProvider = new CDataDictionaryProvider val codec = CompressionCodec.createCodec(SparkEnv.get.conf) val cbbos = new ChunkedByteBufferOutputStream(1024 * 1024, ByteBuffer.allocate) val out = new DataOutputStream(codec.compressedOutputStream(cbbos)) val (fieldVectors, batchProviderOpt) = getBatchFieldVectors(batch) val root = new VectorSchemaRoot(fieldVectors.asJava) val provider = batchProviderOpt.getOrElse(dictionaryProvider) val writer = new ArrowStreamWriter(root, provider, Channels.newChannel(out)) writer.start() writer.writeBatch() root.clear() writer.close() if (out.size() > 0) { (batch.numRows().toLong, cbbos.toChunkedByteBuffer) } else { (batch.numRows().toLong, new ChunkedByteBuffer(Array.empty[ByteBuffer])) } } } /** * Decodes the byte arrays back to ColumnarBatchs and put them into buffer. * * @param bytes * the serialized batches * @param source * the class that calls this method * @return * an iterator of ColumnarBatch */ def decodeBatches(bytes: ChunkedByteBuffer, source: String): Iterator[ColumnarBatch] = { if (bytes.size == 0) { return Iterator.empty } // use Spark's compression codec (LZ4 by default) and not Comet's compression val codec = CompressionCodec.createCodec(SparkEnv.get.conf) val cbbis = bytes.toInputStream() val ins = new DataInputStream(codec.compressedInputStream(cbbis)) // batches are in Arrow IPC format new ArrowReaderIterator(Channels.newChannel(ins), source) } /** * Coalesces many small Arrow IPC batches into a single batch for broadcasting. * * Why this is necessary: The broadcast exchange collects shuffle output by calling * getByteArrayRdd, which serializes each ColumnarBatch independently into its own * ChunkedByteBuffer. The shuffle reader (CometBlockStoreShuffleReader) produces one * ColumnarBatch per shuffle block, and there is one block per writer task per output partition. * So with W writer tasks and P output partitions, the broadcast collects up to W * P tiny * batches. For example, with 400 writer tasks and 500 partitions, 1M rows would arrive as ~200K * batches of ~5 rows each. * * Without coalescing, every consumer task in the broadcast join would independently deserialize * all of these tiny Arrow IPC streams, paying per-stream overhead (schema parsing, buffer * allocation) for each one. With coalescing, we decode and append all batches into one * VectorSchemaRoot on the driver, then re-serialize once. Each consumer task then deserializes * a single Arrow IPC stream. */ def coalesceBroadcastBatches( input: Iterator[ChunkedByteBuffer]): (Array[ChunkedByteBuffer], Long, Long) = { val buffers = input.filterNot(_.size == 0).toArray if (buffers.isEmpty) { return (Array.empty, 0L, 0L) } val allocator = org.apache.comet.CometArrowAllocator .newChildAllocator("broadcast-coalesce", 0, Long.MaxValue) try { var targetRoot: VectorSchemaRoot = null var totalRows = 0L var batchCount = 0 val codec = CompressionCodec.createCodec(SparkEnv.get.conf) try { for (bytes <- buffers) { val compressedInputStream = new DataInputStream(codec.compressedInputStream(bytes.toInputStream())) val reader = new ArrowStreamReader(Channels.newChannel(compressedInputStream), allocator) try { // Comet decodes dictionaries during execution, so this shouldn't happen. // If it does, fall back to the original uncoalesced buffers because each // partition can have a different dictionary, and appending index vectors // would silently mix indices from incompatible dictionaries. if (!reader.getDictionaryVectors.isEmpty) { logWarning( "Unexpected dictionary-encoded column during BroadcastExchange coalescing; " + "skipping coalesce") reader.close() if (targetRoot != null) { targetRoot.close() targetRoot = null } return (buffers, 0L, 0L) } while (reader.loadNextBatch()) { val sourceRoot = reader.getVectorSchemaRoot if (targetRoot == null) { targetRoot = VectorSchemaRoot.create(sourceRoot.getSchema, allocator) targetRoot.allocateNew() } VectorSchemaRootAppender.append(targetRoot, sourceRoot) totalRows += sourceRoot.getRowCount batchCount += 1 } } finally { reader.close() } } if (targetRoot == null) { return (Array.empty, 0L, 0L) } assert( targetRoot.getRowCount.toLong == totalRows, s"Row count mismatch after coalesce: ${targetRoot.getRowCount} != $totalRows") logInfo(s"Coalesced $batchCount broadcast batches into 1 ($totalRows rows)") val outputStream = new ChunkedByteBufferOutputStream(1024 * 1024, ByteBuffer.allocate) val compressedOutputStream = new DataOutputStream(codec.compressedOutputStream(outputStream)) val writer = new ArrowStreamWriter(targetRoot, null, Channels.newChannel(compressedOutputStream)) try { writer.start() writer.writeBatch() } finally { writer.close() } (Array(outputStream.toChunkedByteBuffer), batchCount.toLong, totalRows) } finally { if (targetRoot != null) { targetRoot.close() } } } finally { allocator.close() } } def getBatchFieldVectors( batch: ColumnarBatch): (Seq[FieldVector], Option[DictionaryProvider]) = { var provider: Option[DictionaryProvider] = None val fieldVectors = (0 until batch.numCols()).map { index => batch.column(index) match { case a: CometVector => val valueVector = a.getValueVector if (valueVector.getField.getDictionary != null) { if (provider.isEmpty) { provider = Some(a.getDictionaryProvider) } } getFieldVector(valueVector, "serialize") case c => throw new SparkException( s"Comet execution only takes Arrow Arrays, but got ${c.getClass}. " + "This typically happens when a Comet scan falls back to Spark due to unsupported " + "data types (e.g., complex types like structs, arrays, or maps). " + "To resolve this, you can: " + "(1) enable spark.comet.scan.allowIncompatible=true to use a compatible native " + "scan variant, or " + "(2) enable spark.comet.convert.parquet.enabled=true to convert Spark Parquet " + "data to Arrow format automatically.") } } (fieldVectors, provider) } def getFieldVector(valueVector: ValueVector, reason: String): FieldVector = { valueVector match { case v @ (_: BitVector | _: TinyIntVector | _: SmallIntVector | _: IntVector | _: BigIntVector | _: Float4Vector | _: Float8Vector | _: VarCharVector | _: DecimalVector | _: DateDayVector | _: TimeStampMicroTZVector | _: VarBinaryVector | _: FixedSizeBinaryVector | _: TimeStampMicroVector | _: StructVector | _: ListVector | _: MapVector | _: NullVector) => v.asInstanceOf[FieldVector] case _ => throw new SparkException(s"Unsupported Arrow Vector for $reason: ${valueVector.getClass}") } } } ================================================ FILE: common/src/main/spark-3.4/org/apache/comet/shims/ShimBatchReader.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.shims import org.apache.spark.paths.SparkPath import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.execution.datasources.PartitionedFile object ShimBatchReader { def newPartitionedFile(partitionValues: InternalRow, file: String): PartitionedFile = PartitionedFile( partitionValues, SparkPath.fromPathString(file), -1, // -1 means we read the entire file -1, Array.empty[String], 0, 0) } ================================================ FILE: common/src/main/spark-3.4/org/apache/comet/shims/ShimFileFormat.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.shims import org.apache.spark.sql.execution.datasources.{FileFormat, RowIndexUtil} import org.apache.spark.sql.types.StructType object ShimFileFormat { // A name for a temporary column that holds row indexes computed by the file format reader // until they can be placed in the _metadata struct. val ROW_INDEX_TEMPORARY_COLUMN_NAME: String = FileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME def findRowIndexColumnIndexInSchema(sparkSchema: StructType): Int = RowIndexUtil.findRowIndexColumnIndexInSchema(sparkSchema) } ================================================ FILE: common/src/main/spark-3.4/org/apache/spark/sql/comet/shims/ShimTaskMetrics.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.spark.sql.comet.shims import org.apache.spark.executor.TaskMetrics import org.apache.spark.util.AccumulatorV2 object ShimTaskMetrics { def getTaskAccumulator(taskMetrics: TaskMetrics): Option[AccumulatorV2[_, _]] = taskMetrics.externalAccums.lastOption } ================================================ FILE: common/src/main/spark-3.5/org/apache/comet/shims/ShimBatchReader.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.shims import org.apache.spark.paths.SparkPath import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.execution.datasources.PartitionedFile object ShimBatchReader { def newPartitionedFile(partitionValues: InternalRow, file: String): PartitionedFile = PartitionedFile( partitionValues, SparkPath.fromPathString(file), -1, // -1 means we read the entire file -1, Array.empty[String], 0, 0, Map.empty) } ================================================ FILE: common/src/main/spark-3.5/org/apache/comet/shims/ShimFileFormat.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.shims import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat import org.apache.spark.sql.execution.datasources.parquet.ParquetRowIndexUtil import org.apache.spark.sql.types.StructType object ShimFileFormat { // A name for a temporary column that holds row indexes computed by the file format reader // until they can be placed in the _metadata struct. val ROW_INDEX_TEMPORARY_COLUMN_NAME = ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME def findRowIndexColumnIndexInSchema(sparkSchema: StructType): Int = ParquetRowIndexUtil.findRowIndexColumnIndexInSchema(sparkSchema) } ================================================ FILE: common/src/main/spark-3.5/org/apache/spark/sql/comet/shims/ShimTaskMetrics.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.spark.sql.comet.shims import scala.collection.mutable.ArrayBuffer import org.apache.spark.executor.TaskMetrics import org.apache.spark.util.AccumulatorV2 object ShimTaskMetrics { def getTaskAccumulator(taskMetrics: TaskMetrics): Option[AccumulatorV2[_, _]] = taskMetrics.withExternalAccums(identity[ArrayBuffer[AccumulatorV2[_, _]]](_)).lastOption } ================================================ FILE: common/src/main/spark-3.x/org/apache/comet/shims/CometTypeShim.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.shims import scala.annotation.nowarn import org.apache.spark.sql.types.DataType trait CometTypeShim { @nowarn // Spark 4 feature; stubbed to false in Spark 3.x for compatibility. def isStringCollationType(dt: DataType): Boolean = false } ================================================ FILE: common/src/main/spark-3.x/org/apache/comet/shims/ShimCometConf.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.shims trait ShimCometConf { protected val COMET_SCHEMA_EVOLUTION_ENABLED_DEFAULT = false } ================================================ FILE: common/src/main/spark-4.0/org/apache/comet/shims/CometTypeShim.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.shims import org.apache.spark.sql.internal.types.StringTypeWithCollation import org.apache.spark.sql.types.DataType trait CometTypeShim { def isStringCollationType(dt: DataType): Boolean = dt.isInstanceOf[StringTypeWithCollation] } ================================================ FILE: common/src/main/spark-4.0/org/apache/comet/shims/ShimBatchReader.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.shims import org.apache.spark.paths.SparkPath import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.execution.datasources.PartitionedFile object ShimBatchReader { def newPartitionedFile(partitionValues: InternalRow, file: String): PartitionedFile = PartitionedFile( partitionValues, SparkPath.fromUrlString(file), -1, // -1 means we read the entire file -1, Array.empty[String], 0, 0) } ================================================ FILE: common/src/main/spark-4.0/org/apache/comet/shims/ShimCometConf.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.shims trait ShimCometConf { protected val COMET_SCHEMA_EVOLUTION_ENABLED_DEFAULT = true } ================================================ FILE: common/src/main/spark-4.0/org/apache/comet/shims/ShimFileFormat.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.shims import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat import org.apache.spark.sql.execution.datasources.parquet.ParquetRowIndexUtil import org.apache.spark.sql.types.StructType object ShimFileFormat { // A name for a temporary column that holds row indexes computed by the file format reader // until they can be placed in the _metadata struct. val ROW_INDEX_TEMPORARY_COLUMN_NAME = ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME def findRowIndexColumnIndexInSchema(sparkSchema: StructType): Int = ParquetRowIndexUtil.findRowIndexColumnIndexInSchema(sparkSchema) } ================================================ FILE: common/src/main/spark-4.0/org/apache/spark/sql/comet/shims/ShimTaskMetrics.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.spark.sql.comet.shims import org.apache.spark.executor.TaskMetrics import org.apache.spark.util.AccumulatorV2 object ShimTaskMetrics { def getTaskAccumulator(taskMetrics: TaskMetrics): Option[AccumulatorV2[_, _]] = taskMetrics._externalAccums.lastOption } ================================================ FILE: common/src/test/java/org/apache/comet/parquet/TestColumnReader.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.parquet; import org.junit.Test; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.FixedSizeBinaryVector; import org.apache.arrow.vector.IntVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.comet.vector.CometPlainVector; import org.apache.comet.vector.CometVector; import static org.junit.Assert.*; public class TestColumnReader { @Test public void testIsFixedLength() { BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); ValueVector vv = new IntVector("v1", allocator); CometVector vector = new CometPlainVector(vv, false); assertTrue(vector.isFixedLength()); vv = new FixedSizeBinaryVector("v2", allocator, 12); vector = new CometPlainVector(vv, false); assertTrue(vector.isFixedLength()); vv = new VarBinaryVector("v3", allocator); vector = new CometPlainVector(vv, false); assertFalse(vector.isFixedLength()); } } ================================================ FILE: common/src/test/java/org/apache/comet/parquet/TestCometInputFile.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.parquet; import org.junit.Assert; import org.junit.Test; public class TestCometInputFile { @Test public void testIsAtLeastHadoop33() { Assert.assertTrue(CometInputFile.isAtLeastHadoop33("3.3.0")); Assert.assertTrue(CometInputFile.isAtLeastHadoop33("3.4.0-SNAPSHOT")); Assert.assertTrue(CometInputFile.isAtLeastHadoop33("3.12.5")); Assert.assertTrue(CometInputFile.isAtLeastHadoop33("3.20.6.4-xyz")); Assert.assertFalse(CometInputFile.isAtLeastHadoop33("2.7.2")); Assert.assertFalse(CometInputFile.isAtLeastHadoop33("2.7.3-SNAPSHOT")); Assert.assertFalse(CometInputFile.isAtLeastHadoop33("2.7")); Assert.assertFalse(CometInputFile.isAtLeastHadoop33("2")); Assert.assertFalse(CometInputFile.isAtLeastHadoop33("3")); Assert.assertFalse(CometInputFile.isAtLeastHadoop33("3.2")); Assert.assertFalse(CometInputFile.isAtLeastHadoop33("3.0.2.5-abc")); Assert.assertFalse(CometInputFile.isAtLeastHadoop33("3.1.2-test")); Assert.assertFalse(CometInputFile.isAtLeastHadoop33("3-SNAPSHOT")); Assert.assertFalse(CometInputFile.isAtLeastHadoop33("3.2-SNAPSHOT")); } } ================================================ FILE: common/src/test/java/org/apache/comet/parquet/TestFileReader.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.parquet; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.lang.reflect.Method; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.*; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.parquet.HadoopReadOptions; import org.apache.parquet.ParquetReadOptions; import org.apache.parquet.bytes.BytesInput; import org.apache.parquet.bytes.BytesUtils; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.Encoding; import org.apache.parquet.column.page.DataPage; import org.apache.parquet.column.page.DataPageV1; import org.apache.parquet.column.page.DataPageV2; import org.apache.parquet.column.page.DictionaryPage; import org.apache.parquet.column.page.PageReadStore; import org.apache.parquet.column.page.PageReader; import org.apache.parquet.column.statistics.BinaryStatistics; import org.apache.parquet.column.statistics.Statistics; import org.apache.parquet.column.values.bloomfilter.BlockSplitBloomFilter; import org.apache.parquet.column.values.bloomfilter.BloomFilter; import org.apache.parquet.filter2.predicate.FilterApi; import org.apache.parquet.filter2.predicate.FilterPredicate; import org.apache.parquet.filter2.predicate.Operators; import org.apache.parquet.hadoop.ParquetFileWriter; import org.apache.parquet.hadoop.ParquetInputFormat; import org.apache.parquet.hadoop.metadata.*; import org.apache.parquet.hadoop.util.HadoopInputFile; import org.apache.parquet.internal.column.columnindex.BoundaryOrder; import org.apache.parquet.internal.column.columnindex.ColumnIndex; import org.apache.parquet.internal.column.columnindex.OffsetIndex; import org.apache.parquet.io.InputFile; import org.apache.parquet.io.api.Binary; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.MessageTypeParser; import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; import org.apache.parquet.schema.Types; import org.apache.comet.CometConf; import static org.apache.parquet.column.Encoding.*; import static org.apache.parquet.format.converter.ParquetMetadataConverter.MAX_STATS_SIZE; import static org.junit.Assert.*; import static org.junit.Assert.assertEquals; import static org.apache.comet.parquet.TypeUtil.isSpark40Plus; @SuppressWarnings("deprecation") public class TestFileReader { private static final MessageType SCHEMA = MessageTypeParser.parseMessageType( "" + "message m {" + " required group a {" + " required binary b;" + " }" + " required group c {" + " required int64 d;" + " }" + "}"); private static final MessageType SCHEMA2 = MessageTypeParser.parseMessageType( "" + "message root { " + "required int32 id;" + "required binary name(UTF8); " + "required int32 num; " + "required binary comment(UTF8);" + "}"); private static final MessageType PROJECTED_SCHEMA2 = MessageTypeParser.parseMessageType( "" + "message root { " + "required int32 id;" + "required binary name(UTF8); " + "required binary comment(UTF8);" + "}"); private static final String[] PATH1 = {"a", "b"}; private static final ColumnDescriptor C1 = SCHEMA.getColumnDescription(PATH1); private static final String[] PATH2 = {"c", "d"}; private static final ColumnDescriptor C2 = SCHEMA.getColumnDescription(PATH2); private static final byte[] BYTES1 = {0, 1, 2, 3}; private static final byte[] BYTES2 = {1, 2, 3, 4}; private static final byte[] BYTES3 = {2, 3, 4, 5}; private static final byte[] BYTES4 = {3, 4, 5, 6}; private static final CompressionCodecName CODEC = CompressionCodecName.UNCOMPRESSED; private static final org.apache.parquet.column.statistics.Statistics EMPTY_STATS = org.apache.parquet.column.statistics.Statistics.getBuilderForReading( Types.required(PrimitiveTypeName.BINARY).named("test_binary")) .build(); @Rule public final TemporaryFolder temp = new TemporaryFolder(); @Test public void testEnableReadParallel() { Configuration configuration = new Configuration(); ReadOptions options = ReadOptions.builder(configuration).build(); assertFalse(FileReader.shouldReadParallel(options, "hdfs")); assertFalse(FileReader.shouldReadParallel(options, "file")); assertFalse(FileReader.shouldReadParallel(options, null)); assertTrue(FileReader.shouldReadParallel(options, "s3a")); options = ReadOptions.builder(configuration).enableParallelIO(false).build(); assertFalse(FileReader.shouldReadParallel(options, "s3a")); } @Test public void testReadWrite() throws Exception { File testFile = temp.newFile(); testFile.delete(); Path path = new Path(testFile.toURI()); Configuration configuration = new Configuration(); // Start a Parquet file with 2 row groups, each with 2 column chunks ParquetFileWriter w = new ParquetFileWriter(configuration, SCHEMA, path); w.start(); w.startBlock(3); w.startColumn(C1, 5, CODEC); long c1Starts = w.getPos(); long c1p1Starts = w.getPos(); w.writeDataPage(2, 4, BytesInput.from(BYTES1), EMPTY_STATS, 2, RLE, RLE, PLAIN); w.writeDataPage(3, 4, BytesInput.from(BYTES1), EMPTY_STATS, 3, RLE, RLE, PLAIN); w.endColumn(); long c1Ends = w.getPos(); w.startColumn(C2, 6, CODEC); long c2Starts = w.getPos(); w.writeDictionaryPage(new DictionaryPage(BytesInput.from(BYTES2), 4, RLE_DICTIONARY)); long c2p1Starts = w.getPos(); w.writeDataPage(2, 4, BytesInput.from(BYTES2), EMPTY_STATS, 2, RLE, RLE, PLAIN); w.writeDataPage(3, 4, BytesInput.from(BYTES2), EMPTY_STATS, 3, RLE, RLE, PLAIN); w.writeDataPage(1, 4, BytesInput.from(BYTES2), EMPTY_STATS, 1, RLE, RLE, PLAIN); w.endColumn(); long c2Ends = w.getPos(); w.endBlock(); w.startBlock(4); w.startColumn(C1, 7, CODEC); w.writeDataPage(7, 4, BytesInput.from(BYTES3), EMPTY_STATS, 7, RLE, RLE, PLAIN); w.endColumn(); w.startColumn(C2, 8, CODEC); w.writeDataPage(8, 4, BytesInput.from(BYTES4), EMPTY_STATS, 8, RLE, RLE, PLAIN); w.endColumn(); w.endBlock(); w.end(new HashMap<>()); InputFile file = HadoopInputFile.fromPath(path, configuration); ParquetReadOptions options = ParquetReadOptions.builder().build(); ReadOptions cometOptions = ReadOptions.builder(configuration).build(); try (FileReader reader = new FileReader(file, options, cometOptions)) { ParquetMetadata readFooter = reader.getFooter(); assertEquals("footer: " + readFooter, 2, readFooter.getBlocks().size()); BlockMetaData rowGroup = readFooter.getBlocks().get(0); assertEquals(c1Ends - c1Starts, rowGroup.getColumns().get(0).getTotalSize()); assertEquals(c2Ends - c2Starts, rowGroup.getColumns().get(1).getTotalSize()); assertEquals(c2Ends - c1Starts, rowGroup.getTotalByteSize()); assertEquals(c1Starts, rowGroup.getColumns().get(0).getStartingPos()); assertEquals(0, rowGroup.getColumns().get(0).getDictionaryPageOffset()); assertEquals(c1p1Starts, rowGroup.getColumns().get(0).getFirstDataPageOffset()); assertEquals(c2Starts, rowGroup.getColumns().get(1).getStartingPos()); assertEquals(c2Starts, rowGroup.getColumns().get(1).getDictionaryPageOffset()); assertEquals(c2p1Starts, rowGroup.getColumns().get(1).getFirstDataPageOffset()); HashSet expectedEncoding = new HashSet<>(); expectedEncoding.add(PLAIN); expectedEncoding.add(RLE); assertEquals(expectedEncoding, rowGroup.getColumns().get(0).getEncodings()); } // read first block of col #1 try (FileReader r = new FileReader(file, options, cometOptions)) { r.setRequestedSchema(Arrays.asList(SCHEMA.getColumnDescription(PATH1))); PageReadStore pages = r.readNextRowGroup(); assertEquals(3, pages.getRowCount()); validateContains(pages, PATH1, 2, BytesInput.from(BYTES1)); validateContains(pages, PATH1, 3, BytesInput.from(BYTES1)); assertTrue(r.skipNextRowGroup()); assertNull(r.readNextRowGroup()); } // read all blocks of col #1 and #2 try (FileReader r = new FileReader(file, options, cometOptions)) { r.setRequestedSchema( Arrays.asList(SCHEMA.getColumnDescription(PATH1), SCHEMA.getColumnDescription(PATH2))); PageReadStore pages = r.readNextRowGroup(); assertEquals(3, pages.getRowCount()); validateContains(pages, PATH1, 2, BytesInput.from(BYTES1)); validateContains(pages, PATH1, 3, BytesInput.from(BYTES1)); validateContains(pages, PATH2, 2, BytesInput.from(BYTES2)); validateContains(pages, PATH2, 3, BytesInput.from(BYTES2)); validateContains(pages, PATH2, 1, BytesInput.from(BYTES2)); pages = r.readNextRowGroup(); assertEquals(4, pages.getRowCount()); validateContains(pages, PATH1, 7, BytesInput.from(BYTES3)); validateContains(pages, PATH2, 8, BytesInput.from(BYTES4)); assertNull(r.readNextRowGroup()); } } @Test public void testBloomFilterReadWrite() throws Exception { MessageType schema = MessageTypeParser.parseMessageType("message test { required binary foo; }"); File testFile = temp.newFile(); testFile.delete(); Path path = new Path(testFile.toURI()); Configuration configuration = new Configuration(); configuration.set("parquet.bloom.filter.column.names", "foo"); String[] colPath = {"foo"}; ColumnDescriptor col = schema.getColumnDescription(colPath); BinaryStatistics stats1 = new BinaryStatistics(); ParquetFileWriter w = new ParquetFileWriter(configuration, schema, path); w.start(); w.startBlock(3); w.startColumn(col, 5, CODEC); w.writeDataPage(2, 4, BytesInput.from(BYTES1), stats1, 2, RLE, RLE, PLAIN); w.writeDataPage(3, 4, BytesInput.from(BYTES1), stats1, 2, RLE, RLE, PLAIN); w.endColumn(); BloomFilter blockSplitBloomFilter = new BlockSplitBloomFilter(0); blockSplitBloomFilter.insertHash(blockSplitBloomFilter.hash(Binary.fromString("hello"))); blockSplitBloomFilter.insertHash(blockSplitBloomFilter.hash(Binary.fromString("world"))); addBloomFilter(w, "foo", blockSplitBloomFilter); w.endBlock(); w.end(new HashMap<>()); InputFile file = HadoopInputFile.fromPath(path, configuration); ParquetReadOptions options = ParquetReadOptions.builder().build(); ReadOptions cometOptions = ReadOptions.builder(configuration).build(); try (FileReader r = new FileReader(file, options, cometOptions)) { ParquetMetadata footer = r.getFooter(); r.setRequestedSchema(Arrays.asList(schema.getColumnDescription(colPath))); BloomFilterReader bloomFilterReader = new BloomFilterReader( footer.getBlocks().get(0), r.getFileMetaData().getFileDecryptor(), r.getInputStream()); BloomFilter bloomFilter = bloomFilterReader.readBloomFilter(footer.getBlocks().get(0).getColumns().get(0)); assertTrue(bloomFilter.findHash(blockSplitBloomFilter.hash(Binary.fromString("hello")))); assertTrue(bloomFilter.findHash(blockSplitBloomFilter.hash(Binary.fromString("world")))); } } @Test public void testReadWriteDataPageV2() throws Exception { File testFile = temp.newFile(); testFile.delete(); Path path = new Path(testFile.toURI()); Configuration configuration = new Configuration(); ParquetFileWriter w = new ParquetFileWriter(configuration, SCHEMA, path); w.start(); w.startBlock(14); BytesInput repLevels = BytesInput.fromInt(2); BytesInput defLevels = BytesInput.fromInt(1); BytesInput data = BytesInput.fromInt(3); BytesInput data2 = BytesInput.fromInt(10); org.apache.parquet.column.statistics.Statistics statsC1P1 = createStatistics("s", "z", C1); org.apache.parquet.column.statistics.Statistics statsC1P2 = createStatistics("b", "d", C1); w.startColumn(C1, 6, CODEC); long c1Starts = w.getPos(); w.writeDataPageV2(4, 1, 3, repLevels, defLevels, PLAIN, data, 4, statsC1P1); w.writeDataPageV2(3, 0, 3, repLevels, defLevels, PLAIN, data, 4, statsC1P2); w.endColumn(); long c1Ends = w.getPos(); w.startColumn(C2, 5, CODEC); long c2Starts = w.getPos(); w.writeDataPageV2(5, 2, 3, repLevels, defLevels, PLAIN, data2, 4, EMPTY_STATS); w.writeDataPageV2(2, 0, 2, repLevels, defLevels, PLAIN, data2, 4, EMPTY_STATS); w.endColumn(); long c2Ends = w.getPos(); w.endBlock(); w.end(new HashMap<>()); InputFile file = HadoopInputFile.fromPath(path, configuration); ParquetReadOptions options = ParquetReadOptions.builder().build(); ReadOptions cometOptions = ReadOptions.builder(configuration).build(); try (FileReader reader = new FileReader(file, options, cometOptions)) { ParquetMetadata footer = reader.getFooter(); assertEquals("footer: " + footer, 1, footer.getBlocks().size()); assertEquals(c1Ends - c1Starts, footer.getBlocks().get(0).getColumns().get(0).getTotalSize()); assertEquals(c2Ends - c2Starts, footer.getBlocks().get(0).getColumns().get(1).getTotalSize()); assertEquals(c2Ends - c1Starts, footer.getBlocks().get(0).getTotalByteSize()); // check for stats org.apache.parquet.column.statistics.Statistics expectedStats = createStatistics("b", "z", C1); assertStatsValuesEqual( expectedStats, footer.getBlocks().get(0).getColumns().get(0).getStatistics()); HashSet expectedEncoding = new HashSet<>(); expectedEncoding.add(PLAIN); assertEquals(expectedEncoding, footer.getBlocks().get(0).getColumns().get(0).getEncodings()); } try (FileReader r = new FileReader(file, options, cometOptions)) { r.setRequestedSchema( Arrays.asList(SCHEMA.getColumnDescription(PATH1), SCHEMA.getColumnDescription(PATH2))); PageReadStore pages = r.readNextRowGroup(); assertEquals(14, pages.getRowCount()); validateV2Page( pages, PATH1, 3, 4, 1, repLevels.toByteArray(), defLevels.toByteArray(), data.toByteArray(), 12); validateV2Page( pages, PATH1, 3, 3, 0, repLevels.toByteArray(), defLevels.toByteArray(), data.toByteArray(), 12); validateV2Page( pages, PATH2, 3, 5, 2, repLevels.toByteArray(), defLevels.toByteArray(), data2.toByteArray(), 12); validateV2Page( pages, PATH2, 2, 2, 0, repLevels.toByteArray(), defLevels.toByteArray(), data2.toByteArray(), 12); assertNull(r.readNextRowGroup()); } } @Test public void testColumnIndexFilter() throws Exception { File testFile = temp.newFile(); testFile.delete(); Path path = new Path(testFile.toURI()); Configuration configuration = new Configuration(); ParquetFileWriter w = new ParquetFileWriter(configuration, SCHEMA, path); w.start(); w.startBlock(4); w.startColumn(C1, 7, CODEC); w.writeDataPage(2, 4, BytesInput.from(BYTES1), EMPTY_STATS, 2, RLE, RLE, PLAIN); w.writeDataPage(2, 4, BytesInput.from(BYTES2), EMPTY_STATS, 2, RLE, RLE, PLAIN); w.endColumn(); w.startColumn(C2, 8, CODEC); // the first page contains one matching record w.writeDataPage(1, 4, BytesInput.from(BYTES3), statsC2(2L), 1, RLE, RLE, PLAIN); // all the records of the second page are larger than 2, so should be filtered out w.writeDataPage(3, 4, BytesInput.from(BYTES4), statsC2(3L, 4L, 5L), 3, RLE, RLE, PLAIN); w.endColumn(); w.endBlock(); w.startBlock(4); w.startColumn(C1, 7, CODEC); w.writeDataPage(2, 4, BytesInput.from(BYTES1), EMPTY_STATS, 2, RLE, RLE, PLAIN); w.writeDataPage(2, 4, BytesInput.from(BYTES2), EMPTY_STATS, 2, RLE, RLE, PLAIN); w.endColumn(); w.startColumn(C2, 8, CODEC); // the first page should be filtered out w.writeDataPage(1, 4, BytesInput.from(BYTES3), statsC2(4L), 1, RLE, RLE, PLAIN); // the second page will be read since it contains matching record w.writeDataPage(3, 4, BytesInput.from(BYTES4), statsC2(0L, 1L, 3L), 3, RLE, RLE, PLAIN); w.endColumn(); w.endBlock(); w.end(new HashMap<>()); // set a simple equality filter in the ParquetInputFormat Operators.LongColumn c2 = FilterApi.longColumn("c.d"); FilterPredicate p = FilterApi.eq(c2, 2L); ParquetInputFormat.setFilterPredicate(configuration, p); InputFile file = HadoopInputFile.fromPath(path, configuration); ParquetReadOptions options = HadoopReadOptions.builder(configuration).build(); ReadOptions cometOptions = ReadOptions.builder(configuration).build(); try (FileReader r = new FileReader(file, options, cometOptions)) { assertEquals(4, r.getFilteredRecordCount()); PageReadStore readStore = r.readNextFilteredRowGroup(); PageReader c1Reader = readStore.getPageReader(C1); List c1Pages = new ArrayList<>(); DataPage page; while ((page = c1Reader.readPage()) != null) { c1Pages.add(page); } // second page of c1 should be filtered out assertEquals(1, c1Pages.size()); validatePage(c1Pages.get(0), 2, BytesInput.from(BYTES1)); PageReader c2Reader = readStore.getPageReader(C2); List c2Pages = new ArrayList<>(); while ((page = c2Reader.readPage()) != null) { c2Pages.add(page); } assertEquals(1, c2Pages.size()); validatePage(c2Pages.get(0), 1, BytesInput.from(BYTES3)); // test the second row group readStore = r.readNextFilteredRowGroup(); assertNotNull(readStore); c1Reader = readStore.getPageReader(C1); c1Pages.clear(); while ((page = c1Reader.readPage()) != null) { c1Pages.add(page); } // all pages of c1 should be retained assertEquals(2, c1Pages.size()); validatePage(c1Pages.get(0), 2, BytesInput.from(BYTES1)); validatePage(c1Pages.get(1), 2, BytesInput.from(BYTES2)); c2Reader = readStore.getPageReader(C2); c2Pages.clear(); while ((page = c2Reader.readPage()) != null) { c2Pages.add(page); } assertEquals(1, c2Pages.size()); validatePage(c2Pages.get(0), 3, BytesInput.from(BYTES4)); } } @Test public void testColumnIndexReadWrite() throws Exception { File testFile = temp.newFile(); testFile.delete(); Path path = new Path(testFile.toURI()); Configuration configuration = new Configuration(); ParquetFileWriter w = new ParquetFileWriter(configuration, SCHEMA, path); w.start(); w.startBlock(4); w.startColumn(C1, 7, CODEC); w.writeDataPage(7, 4, BytesInput.from(BYTES3), EMPTY_STATS, RLE, RLE, PLAIN); w.endColumn(); w.startColumn(C2, 8, CODEC); w.writeDataPage(8, 4, BytesInput.from(BYTES4), EMPTY_STATS, RLE, RLE, PLAIN); w.endColumn(); w.endBlock(); w.startBlock(4); w.startColumn(C1, 5, CODEC); long c1p1Starts = w.getPos(); w.writeDataPage( 2, 4, BytesInput.from(BYTES1), statsC1(null, Binary.fromString("aaa")), 1, RLE, RLE, PLAIN); long c1p2Starts = w.getPos(); w.writeDataPage( 3, 4, BytesInput.from(BYTES1), statsC1(Binary.fromString("bbb"), Binary.fromString("ccc")), 3, RLE, RLE, PLAIN); w.endColumn(); long c1Ends = w.getPos(); w.startColumn(C2, 6, CODEC); long c2p1Starts = w.getPos(); w.writeDataPage(2, 4, BytesInput.from(BYTES2), statsC2(117L, 100L), 1, RLE, RLE, PLAIN); long c2p2Starts = w.getPos(); w.writeDataPage(3, 4, BytesInput.from(BYTES2), statsC2(null, null, null), 2, RLE, RLE, PLAIN); long c2p3Starts = w.getPos(); w.writeDataPage(1, 4, BytesInput.from(BYTES2), statsC2(0L), 1, RLE, RLE, PLAIN); w.endColumn(); long c2Ends = w.getPos(); w.endBlock(); w.startBlock(4); w.startColumn(C1, 7, CODEC); w.writeDataPage( 7, 4, BytesInput.from(BYTES3), // Creating huge stats so the column index will reach the limit and won't be written statsC1( Binary.fromConstantByteArray(new byte[(int) MAX_STATS_SIZE]), Binary.fromConstantByteArray(new byte[1])), 4, RLE, RLE, PLAIN); w.endColumn(); w.startColumn(C2, 8, CODEC); w.writeDataPage(8, 4, BytesInput.from(BYTES4), EMPTY_STATS, RLE, RLE, PLAIN); w.endColumn(); w.endBlock(); w.end(new HashMap<>()); InputFile file = HadoopInputFile.fromPath(path, configuration); ParquetReadOptions options = ParquetReadOptions.builder().build(); ReadOptions cometOptions = ReadOptions.builder(configuration).build(); try (FileReader reader = new FileReader(file, options, cometOptions)) { ParquetMetadata footer = reader.getFooter(); assertEquals(3, footer.getBlocks().size()); BlockMetaData blockMeta = footer.getBlocks().get(1); assertEquals(2, blockMeta.getColumns().size()); ColumnIndexReader indexReader = reader.getColumnIndexReader(1); ColumnIndex columnIndex = indexReader.readColumnIndex(blockMeta.getColumns().get(0)); assertEquals(BoundaryOrder.ASCENDING, columnIndex.getBoundaryOrder()); assertEquals(Arrays.asList(1L, 0L), columnIndex.getNullCounts()); assertEquals(Arrays.asList(false, false), columnIndex.getNullPages()); List minValues = columnIndex.getMinValues(); assertEquals(2, minValues.size()); List maxValues = columnIndex.getMaxValues(); assertEquals(2, maxValues.size()); assertEquals("aaa", new String(minValues.get(0).array(), StandardCharsets.UTF_8)); assertEquals("aaa", new String(maxValues.get(0).array(), StandardCharsets.UTF_8)); assertEquals("bbb", new String(minValues.get(1).array(), StandardCharsets.UTF_8)); assertEquals("ccc", new String(maxValues.get(1).array(), StandardCharsets.UTF_8)); columnIndex = indexReader.readColumnIndex(blockMeta.getColumns().get(1)); assertEquals(BoundaryOrder.DESCENDING, columnIndex.getBoundaryOrder()); assertEquals(Arrays.asList(0L, 3L, 0L), columnIndex.getNullCounts()); assertEquals(Arrays.asList(false, true, false), columnIndex.getNullPages()); minValues = columnIndex.getMinValues(); assertEquals(3, minValues.size()); maxValues = columnIndex.getMaxValues(); assertEquals(3, maxValues.size()); assertEquals(100, BytesUtils.bytesToLong(minValues.get(0).array())); assertEquals(117, BytesUtils.bytesToLong(maxValues.get(0).array())); assertEquals(0, minValues.get(1).array().length); assertEquals(0, maxValues.get(1).array().length); assertEquals(0, BytesUtils.bytesToLong(minValues.get(2).array())); assertEquals(0, BytesUtils.bytesToLong(maxValues.get(2).array())); OffsetIndex offsetIndex = indexReader.readOffsetIndex(blockMeta.getColumns().get(0)); assertEquals(2, offsetIndex.getPageCount()); assertEquals(c1p1Starts, offsetIndex.getOffset(0)); assertEquals(c1p2Starts, offsetIndex.getOffset(1)); assertEquals(c1p2Starts - c1p1Starts, offsetIndex.getCompressedPageSize(0)); assertEquals(c1Ends - c1p2Starts, offsetIndex.getCompressedPageSize(1)); assertEquals(0, offsetIndex.getFirstRowIndex(0)); assertEquals(1, offsetIndex.getFirstRowIndex(1)); offsetIndex = indexReader.readOffsetIndex(blockMeta.getColumns().get(1)); assertEquals(3, offsetIndex.getPageCount()); assertEquals(c2p1Starts, offsetIndex.getOffset(0)); assertEquals(c2p2Starts, offsetIndex.getOffset(1)); assertEquals(c2p3Starts, offsetIndex.getOffset(2)); assertEquals(c2p2Starts - c2p1Starts, offsetIndex.getCompressedPageSize(0)); assertEquals(c2p3Starts - c2p2Starts, offsetIndex.getCompressedPageSize(1)); assertEquals(c2Ends - c2p3Starts, offsetIndex.getCompressedPageSize(2)); assertEquals(0, offsetIndex.getFirstRowIndex(0)); assertEquals(1, offsetIndex.getFirstRowIndex(1)); assertEquals(3, offsetIndex.getFirstRowIndex(2)); if (!isSpark40Plus()) { // TODO: https://github.com/apache/datafusion-comet/issues/1948 assertNull(indexReader.readColumnIndex(footer.getBlocks().get(2).getColumns().get(0))); } } } // Test reader with merging of scan ranges enabled @Test public void testWriteReadMergeScanRange() throws Throwable { Configuration conf = new Configuration(); conf.set(CometConf.COMET_IO_MERGE_RANGES().key(), Boolean.toString(true)); // Set the merge range delta so small that ranges do not get merged conf.set(CometConf.COMET_IO_MERGE_RANGES_DELTA().key(), Integer.toString(1024)); testReadWrite(conf, 2, 1024); // Set the merge range delta so large that all ranges get merged conf.set(CometConf.COMET_IO_MERGE_RANGES_DELTA().key(), Integer.toString(1024 * 1024)); testReadWrite(conf, 2, 1024); } // `addBloomFilter` is package-private in Parquet, so this uses reflection to access it private void addBloomFilter(ParquetFileWriter w, String s, BloomFilter filter) throws Exception { Method method = ParquetFileWriter.class.getDeclaredMethod( "addBloomFilter", String.class, BloomFilter.class); method.setAccessible(true); method.invoke(w, s, filter); } private void validateContains(PageReadStore pages, String[] path, int values, BytesInput bytes) throws IOException { PageReader pageReader = pages.getPageReader(SCHEMA.getColumnDescription(path)); DataPage page = pageReader.readPage(); validatePage(page, values, bytes); } private void validatePage(DataPage page, int values, BytesInput bytes) throws IOException { assertEquals(values, page.getValueCount()); assertArrayEquals(bytes.toByteArray(), ((DataPageV1) page).getBytes().toByteArray()); } private void validateV2Page( PageReadStore pages, String[] path, int values, int rows, int nullCount, byte[] repetition, byte[] definition, byte[] data, int uncompressedSize) throws IOException { PageReader pageReader = pages.getPageReader(SCHEMA.getColumnDescription(path)); DataPageV2 page = (DataPageV2) pageReader.readPage(); assertEquals(values, page.getValueCount()); assertEquals(rows, page.getRowCount()); assertEquals(nullCount, page.getNullCount()); assertEquals(uncompressedSize, page.getUncompressedSize()); assertArrayEquals(repetition, page.getRepetitionLevels().toByteArray()); assertArrayEquals(definition, page.getDefinitionLevels().toByteArray()); assertArrayEquals(data, page.getData().toByteArray()); } private Statistics createStatistics(String min, String max, ColumnDescriptor col) { return Statistics.getBuilderForReading(col.getPrimitiveType()) .withMin(Binary.fromString(min).getBytes()) .withMax(Binary.fromString(max).getBytes()) .withNumNulls(0) .build(); } public static void assertStatsValuesEqual(Statistics expected, Statistics actual) { if (expected == actual) { return; } if (expected == null || actual == null) { assertEquals(expected, actual); } Assert.assertArrayEquals(expected.getMaxBytes(), actual.getMaxBytes()); Assert.assertArrayEquals(expected.getMinBytes(), actual.getMinBytes()); Assert.assertEquals(expected.getNumNulls(), actual.getNumNulls()); } private Statistics statsC1(Binary... values) { Statistics stats = Statistics.createStats(C1.getPrimitiveType()); for (Binary value : values) { if (value == null) { stats.incrementNumNulls(); } else { stats.updateStats(value); } } return stats; } /** * Generates arbitrary data for simple schemas, writes the data to a file and also returns the * data. * * @return array of data pages for each column */ private HashMap generateAndWriteData( Configuration configuration, Path path, MessageType schema, int numPages, int numRecordsPerPage) throws IOException { HashMap dataPages = new HashMap<>(); Generator generator = new Generator(); ParquetFileWriter writer = new ParquetFileWriter(configuration, schema, path); writer.start(); writer.startBlock((long) numPages * numRecordsPerPage); for (ColumnDescriptor colDesc : schema.getColumns()) { writer.startColumn(colDesc, (long) numPages * numRecordsPerPage, CODEC); String type = colDesc.getPrimitiveType().getName(); byte[][] allPages = new byte[numPages][]; byte[] data; for (int i = 0; i < numPages; i++) { data = generator.generateValues(numRecordsPerPage, type); writer.writeDataPage( numRecordsPerPage, data.length, BytesInput.from(data), EMPTY_STATS, numRecordsPerPage, RLE, RLE, PLAIN); allPages[i] = data; } dataPages.put(String.join(".", colDesc.getPath()), allPages); writer.endColumn(); } writer.endBlock(); writer.end(new HashMap<>()); return dataPages; } private void readAndValidatePageData( InputFile inputFile, ParquetReadOptions options, ReadOptions cometOptions, MessageType schema, HashMap expected, int expectedValuesPerPage) throws IOException { try (FileReader fileReader = new FileReader(inputFile, options, cometOptions)) { fileReader.setRequestedSchema(schema.getColumns()); PageReadStore pages = fileReader.readNextRowGroup(); for (ColumnDescriptor colDesc : schema.getColumns()) { byte[][] allExpectedPages = expected.get(String.join(".", colDesc.getPath())); PageReader pageReader = pages.getPageReader(colDesc); for (byte[] expectedPage : allExpectedPages) { DataPage page = pageReader.readPage(); validatePage(page, expectedValuesPerPage, BytesInput.from(expectedPage)); } } } } public void testReadWrite(Configuration configuration, int numPages, int numRecordsPerPage) throws Exception { File testFile = temp.newFile(); testFile.delete(); Path path = new Path(testFile.toURI()); HashMap dataPages = generateAndWriteData(configuration, path, SCHEMA2, numPages, numRecordsPerPage); InputFile file = HadoopInputFile.fromPath(path, configuration); ParquetReadOptions options = ParquetReadOptions.builder().build(); ReadOptions cometOptions = ReadOptions.builder(configuration).build(); readAndValidatePageData( file, options, cometOptions, PROJECTED_SCHEMA2, dataPages, numRecordsPerPage); } static class Generator { static Random random = new Random(1729); private static final String ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz -"; private static final int STR_MIN_SIZE = 5; private static final int STR_MAX_SIZE = 30; private byte[] getString(int minSize, int maxSize) { int size = random.nextInt(maxSize - minSize) + minSize; byte[] str = new byte[size]; for (int i = 0; i < size; ++i) { str[i] = (byte) ALPHABET.charAt(random.nextInt(ALPHABET.length())); } return str; } private byte[] generateValues(int numValues, String type) throws IOException { if (type.equals("int32")) { byte[] data = new byte[4 * numValues]; random.nextBytes(data); return data; } else { ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); for (int i = 0; i < numValues; i++) { outputStream.write(getString(STR_MIN_SIZE, STR_MAX_SIZE)); } return outputStream.toByteArray(); } } } private Statistics statsC2(Long... values) { Statistics stats = Statistics.createStats(C2.getPrimitiveType()); for (Long value : values) { if (value == null) { stats.incrementNumNulls(); } else { stats.updateStats(value); } } return stats; } } ================================================ FILE: common/src/test/java/org/apache/comet/parquet/TestUtils.java ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.parquet; import java.util.Collections; import java.util.HashMap; import java.util.Map; import org.junit.Test; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.PrimitiveType; import static org.junit.Assert.*; public class TestUtils { @Test public void testBuildColumnDescriptorWithTimestamp() { Map params = new HashMap<>(); params.put("isAdjustedToUTC", "true"); params.put("unit", "MICROS"); ParquetColumnSpec spec = new ParquetColumnSpec( 10, new String[] {"event_time"}, "INT64", 0, false, 0, 0, "TimestampLogicalTypeAnnotation", params); ColumnDescriptor descriptor = Utils.buildColumnDescriptor(spec); assertNotNull(descriptor); PrimitiveType primitiveType = descriptor.getPrimitiveType(); assertEquals(PrimitiveType.PrimitiveTypeName.INT64, primitiveType.getPrimitiveTypeName()); assertTrue( primitiveType.getLogicalTypeAnnotation() instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation); LogicalTypeAnnotation.TimestampLogicalTypeAnnotation ts = (LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) primitiveType.getLogicalTypeAnnotation(); assertTrue(ts.isAdjustedToUTC()); assertEquals(LogicalTypeAnnotation.TimeUnit.MICROS, ts.getUnit()); } @Test public void testBuildColumnDescriptorWithDecimal() { Map params = new HashMap<>(); params.put("precision", "10"); params.put("scale", "2"); ParquetColumnSpec spec = new ParquetColumnSpec( 11, new String[] {"price"}, "FIXED_LEN_BYTE_ARRAY", 5, false, 0, 0, "DecimalLogicalTypeAnnotation", params); ColumnDescriptor descriptor = Utils.buildColumnDescriptor(spec); PrimitiveType primitiveType = descriptor.getPrimitiveType(); assertEquals( PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY, primitiveType.getPrimitiveTypeName()); LogicalTypeAnnotation.DecimalLogicalTypeAnnotation dec = (LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) primitiveType.getLogicalTypeAnnotation(); assertEquals(10, dec.getPrecision()); assertEquals(2, dec.getScale()); } @Test public void testBuildColumnDescriptorWithIntLogicalType() { Map params = new HashMap<>(); params.put("bitWidth", "32"); params.put("isSigned", "true"); ParquetColumnSpec spec = new ParquetColumnSpec( 12, new String[] {"count"}, "INT32", 0, false, 0, 0, "IntLogicalTypeAnnotation", params); ColumnDescriptor descriptor = Utils.buildColumnDescriptor(spec); PrimitiveType primitiveType = descriptor.getPrimitiveType(); assertEquals(PrimitiveType.PrimitiveTypeName.INT32, primitiveType.getPrimitiveTypeName()); LogicalTypeAnnotation.IntLogicalTypeAnnotation ann = (LogicalTypeAnnotation.IntLogicalTypeAnnotation) primitiveType.getLogicalTypeAnnotation(); assertEquals(32, ann.getBitWidth()); assertTrue(ann.isSigned()); } @Test public void testBuildColumnDescriptorWithStringLogicalType() { ParquetColumnSpec spec = new ParquetColumnSpec( 13, new String[] {"name"}, "BINARY", 0, false, 0, 0, "StringLogicalTypeAnnotation", Collections.emptyMap()); ColumnDescriptor descriptor = Utils.buildColumnDescriptor(spec); PrimitiveType primitiveType = descriptor.getPrimitiveType(); assertEquals(PrimitiveType.PrimitiveTypeName.BINARY, primitiveType.getPrimitiveTypeName()); assertTrue( primitiveType.getLogicalTypeAnnotation() instanceof LogicalTypeAnnotation.StringLogicalTypeAnnotation); } } ================================================ FILE: common/src/test/resources/log4j.properties ================================================ # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Set everything to be logged to the file target/unit-tests.log test.appender=file log4j.rootCategory=INFO, ${test.appender} log4j.appender.file=org.apache.log4j.FileAppender log4j.appender.file.append=true log4j.appender.file.file=target/unit-tests.log log4j.appender.file.layout=org.apache.log4j.PatternLayout log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n # Tests that launch java subprocesses can set the "test.appender" system property to # "console" to avoid having the child process's logs overwrite the unit test's # log file. log4j.appender.console=org.apache.log4j.ConsoleAppender log4j.appender.console.target=System.err log4j.appender.console.layout=org.apache.log4j.PatternLayout log4j.appender.console.layout.ConversionPattern=%t: %m%n # Ignore messages below warning level from Jetty, because it's a bit verbose log4j.logger.org.sparkproject.jetty=WARN ================================================ FILE: common/src/test/resources/log4j2.properties ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # Set everything to be logged to the file target/unit-tests.log rootLogger.level = info rootLogger.appenderRef.file.ref = ${sys:test.appender:-File} appender.file.type = File appender.file.name = File appender.file.fileName = target/unit-tests.log appender.file.layout.type = PatternLayout appender.file.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n # Tests that launch java subprocesses can set the "test.appender" system property to # "console" to avoid having the child process's logs overwrite the unit test's # log file. appender.console.type = Console appender.console.name = console appender.console.target = SYSTEM_ERR appender.console.layout.type = PatternLayout appender.console.layout.pattern = %t: %m%n # Ignore messages below warning level from Jetty, because it's a bit verbose logger.jetty.name = org.sparkproject.jetty logger.jetty.level = warn ================================================ FILE: conf/log4rs.yaml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. appenders: unittest: kind: file path: "target/unit-tests.log" root: level: info appenders: - unittest ================================================ FILE: dev/cargo.config ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. [target.x86_64-apple-darwin] linker = "x86_64-apple-darwin21.4-clang" ar = "x86_64-apple-darwin21.4-ar" [target.aarch64-apple-darwin] linker = "aarch64-apple-darwin21.4-clang" ar = "aarch64-apple-darwin21.4-ar" ================================================ FILE: dev/changelog/0.1.0.md ================================================ # DataFusion Comet 0.1.0 Changelog This release consists of 343 commits from 41 contributors. See credits at the end of this changelog for more information. **Implemented enhancements:** - feat: Add native shuffle and columnar shuffle [#30](https://github.com/apache/datafusion-comet/pull/30) (viirya) - feat: Support Emit::First for SumDecimalGroupsAccumulator [#47](https://github.com/apache/datafusion-comet/pull/47) (viirya) - feat: Nested map support for columnar shuffle [#51](https://github.com/apache/datafusion-comet/pull/51) (viirya) - feat: Support Count(Distinct) and similar aggregation functions [#42](https://github.com/apache/datafusion-comet/pull/42) (huaxingao) - feat: Upgrade to `jni-rs` 0.21 [#50](https://github.com/apache/datafusion-comet/pull/50) (sunchao) - feat: Handle exception thrown from native side [#61](https://github.com/apache/datafusion-comet/pull/61) (sunchao) - feat: Support InSet expression in Comet [#59](https://github.com/apache/datafusion-comet/pull/59) (viirya) - feat: Add `CometNativeException` for exceptions thrown from the native side [#62](https://github.com/apache/datafusion-comet/pull/62) (sunchao) - feat: Add cause to native exception [#63](https://github.com/apache/datafusion-comet/pull/63) (viirya) - feat: Pull based native execution [#69](https://github.com/apache/datafusion-comet/pull/69) (viirya) - feat: Add executeColumnarCollectIterator to CometExec to collect Comet operator result [#71](https://github.com/apache/datafusion-comet/pull/71) (viirya) - feat: Add CometBroadcastExchangeExec to support broadcasting the result of Comet native operator [#80](https://github.com/apache/datafusion-comet/pull/80) (viirya) - feat: Reduce memory consumption when writing sorted shuffle files [#82](https://github.com/apache/datafusion-comet/pull/82) (sunchao) - feat: Add struct/map as unsupported map key/value for columnar shuffle [#84](https://github.com/apache/datafusion-comet/pull/84) (viirya) - feat: Support multiple input sources for CometNativeExec [#87](https://github.com/apache/datafusion-comet/pull/87) (viirya) - feat: Date and timestamp trunc with format array [#94](https://github.com/apache/datafusion-comet/pull/94) (parthchandra) - feat: Support `First`/`Last` aggregate functions [#97](https://github.com/apache/datafusion-comet/pull/97) (huaxingao) - feat: Add support of TakeOrderedAndProjectExec in Comet [#88](https://github.com/apache/datafusion-comet/pull/88) (viirya) - feat: Support Binary in shuffle writer [#106](https://github.com/apache/datafusion-comet/pull/106) (advancedxy) - feat: Add license header by spotless:apply automatically [#110](https://github.com/apache/datafusion-comet/pull/110) (advancedxy) - feat: Add dictionary binary to shuffle writer [#111](https://github.com/apache/datafusion-comet/pull/111) (viirya) - feat: Minimize number of connections used by parallel reader [#126](https://github.com/apache/datafusion-comet/pull/126) (parthchandra) - feat: Support CollectLimit operator [#100](https://github.com/apache/datafusion-comet/pull/100) (advancedxy) - feat: Enable min/max for boolean type [#165](https://github.com/apache/datafusion-comet/pull/165) (huaxingao) - feat: Introduce `CometTaskMemoryManager` and native side memory pool [#83](https://github.com/apache/datafusion-comet/pull/83) (sunchao) - feat: Fix old style names [#201](https://github.com/apache/datafusion-comet/pull/201) (comphead) - feat: enable comet shuffle manager for comet shell [#204](https://github.com/apache/datafusion-comet/pull/204) (zuston) - feat: Support bitwise aggregate functions [#197](https://github.com/apache/datafusion-comet/pull/197) (huaxingao) - feat: Support BloomFilterMightContain expr [#179](https://github.com/apache/datafusion-comet/pull/179) (advancedxy) - feat: Support sort merge join [#178](https://github.com/apache/datafusion-comet/pull/178) (viirya) - feat: Support HashJoin operator [#194](https://github.com/apache/datafusion-comet/pull/194) (viirya) - feat: Remove use of nightly int_roundings feature [#228](https://github.com/apache/datafusion-comet/pull/228) (psvri) - feat: Support Broadcast HashJoin [#211](https://github.com/apache/datafusion-comet/pull/211) (viirya) - feat: Enable Comet broadcast by default [#213](https://github.com/apache/datafusion-comet/pull/213) (viirya) - feat: Add CometRowToColumnar operator [#206](https://github.com/apache/datafusion-comet/pull/206) (advancedxy) - feat: Document the class path / classloader issue with the shuffle manager [#256](https://github.com/apache/datafusion-comet/pull/256) (holdenk) - feat: Port Datafusion Covariance to Comet [#234](https://github.com/apache/datafusion-comet/pull/234) (huaxingao) - feat: Add manual test to calculate spark builtin functions coverage [#263](https://github.com/apache/datafusion-comet/pull/263) (comphead) - feat: Support ANSI mode in CAST from String to Bool [#290](https://github.com/apache/datafusion-comet/pull/290) (andygrove) - feat: Add extended explain info to Comet plan [#255](https://github.com/apache/datafusion-comet/pull/255) (parthchandra) - feat: Improve CometSortMergeJoin statistics [#304](https://github.com/apache/datafusion-comet/pull/304) (planga82) - feat: Add compatibility guide [#316](https://github.com/apache/datafusion-comet/pull/316) (andygrove) - feat: Improve CometHashJoin statistics [#309](https://github.com/apache/datafusion-comet/pull/309) (planga82) - feat: Support Variance [#297](https://github.com/apache/datafusion-comet/pull/297) (huaxingao) - feat: Support murmur3_hash and sha2 family hash functions [#226](https://github.com/apache/datafusion-comet/pull/226) (advancedxy) - feat: Disable cast string to timestamp by default [#337](https://github.com/apache/datafusion-comet/pull/337) (andygrove) - feat: Improve CometBroadcastHashJoin statistics [#339](https://github.com/apache/datafusion-comet/pull/339) (planga82) - feat: Implement Spark-compatible CAST from string to integral types [#307](https://github.com/apache/datafusion-comet/pull/307) (andygrove) - feat: Implement Spark-compatible CAST from string to timestamp types [#335](https://github.com/apache/datafusion-comet/pull/335) (vaibhawvipul) - feat: Implement Spark-compatible CAST float/double to string [#346](https://github.com/apache/datafusion-comet/pull/346) (mattharder91) - feat: Only allow incompatible cast expressions to run in comet if a config is enabled [#362](https://github.com/apache/datafusion-comet/pull/362) (andygrove) - feat: Implement Spark-compatible CAST between integer types [#340](https://github.com/apache/datafusion-comet/pull/340) (ganeshkumar269) - feat: Supports Stddev [#348](https://github.com/apache/datafusion-comet/pull/348) (huaxingao) - feat: Improve cast compatibility tests and docs [#379](https://github.com/apache/datafusion-comet/pull/379) (andygrove) - feat: Implement Spark-compatible CAST from non-integral numeric types to integral types [#399](https://github.com/apache/datafusion-comet/pull/399) (rohitrastogi) - feat: Implement Spark unhex [#342](https://github.com/apache/datafusion-comet/pull/342) (tshauck) - feat: Enable columnar shuffle by default [#250](https://github.com/apache/datafusion-comet/pull/250) (viirya) - feat: Implement Spark-compatible CAST from floating-point/double to decimal [#384](https://github.com/apache/datafusion-comet/pull/384) (vaibhawvipul) - feat: Add logging to explain reasons for Comet not being able to run a query stage natively [#397](https://github.com/apache/datafusion-comet/pull/397) (andygrove) - feat: Add support for TryCast expression in Spark 3.2 and 3.3 [#416](https://github.com/apache/datafusion-comet/pull/416) (vaibhawvipul) - feat: Supports UUID column [#395](https://github.com/apache/datafusion-comet/pull/395) (huaxingao) - feat: correlation support [#456](https://github.com/apache/datafusion-comet/pull/456) (huaxingao) - feat: Implement Spark-compatible CAST from String to Date [#383](https://github.com/apache/datafusion-comet/pull/383) (vidyasankarv) - feat: Add COMET_SHUFFLE_MODE config to control Comet shuffle mode [#460](https://github.com/apache/datafusion-comet/pull/460) (viirya) - feat: Add random row generator in data generator [#451](https://github.com/apache/datafusion-comet/pull/451) (advancedxy) - feat: Add xxhash64 function support [#424](https://github.com/apache/datafusion-comet/pull/424) (advancedxy) - feat: add hex scalar function [#449](https://github.com/apache/datafusion-comet/pull/449) (tshauck) - feat: Add "Comet Fuzz" fuzz-testing utility [#472](https://github.com/apache/datafusion-comet/pull/472) (andygrove) - feat: Use enum to represent CAST eval_mode in expr.proto [#415](https://github.com/apache/datafusion-comet/pull/415) (prashantksharma) - feat: Implement ANSI support for UnaryMinus [#471](https://github.com/apache/datafusion-comet/pull/471) (vaibhawvipul) - feat: Add specific fuzz tests for cast and try_cast and fix NPE found during fuzz testing [#514](https://github.com/apache/datafusion-comet/pull/514) (andygrove) - feat: Add fuzz testing for arithmetic expressions [#519](https://github.com/apache/datafusion-comet/pull/519) (andygrove) - feat: Add HashJoin support for BuildRight [#437](https://github.com/apache/datafusion-comet/pull/437) (viirya) - feat: Fix Comet error message [#544](https://github.com/apache/datafusion-comet/pull/544) (comphead) - feat: Support Ansi mode in abs function [#500](https://github.com/apache/datafusion-comet/pull/500) (planga82) - feat: Enable xxhash64 by default [#583](https://github.com/apache/datafusion-comet/pull/583) (andygrove) - feat: Add experimental support for Apache Spark 3.5.1 [#587](https://github.com/apache/datafusion-comet/pull/587) (andygrove) - feat: add nullOnDivideByZero for Covariance [#564](https://github.com/apache/datafusion-comet/pull/564) (huaxingao) - feat: Implement more efficient version of xxhash64 [#575](https://github.com/apache/datafusion-comet/pull/575) (andygrove) - feat: Enable Spark SQL tests for Spark 3.5.1 [#603](https://github.com/apache/datafusion-comet/pull/603) (andygrove) - feat: Initial support for Window function [#599](https://github.com/apache/datafusion-comet/pull/599) (huaxingao) - feat: IsNaN expression in Comet [#612](https://github.com/apache/datafusion-comet/pull/612) (eejbyfeldt) - feat: Add support for CreateNamedStruct [#620](https://github.com/apache/datafusion-comet/pull/620) (eejbyfeldt) - feat: add cargo machete to remove udeps [#641](https://github.com/apache/datafusion-comet/pull/641) (vaibhawvipul) - feat: Upgrade to DataFusion 40.0.0-rc1 [#644](https://github.com/apache/datafusion-comet/pull/644) (andygrove) - feat: Use unified allocator for execution iterators [#613](https://github.com/apache/datafusion-comet/pull/613) (viirya) - feat: Create new `datafusion-comet-spark-expr` crate containing Spark-compatible DataFusion expressions [#638](https://github.com/apache/datafusion-comet/pull/638) (andygrove) - feat: Move `IfExpr` to `spark-expr` crate [#653](https://github.com/apache/datafusion-comet/pull/653) (andygrove) - feat: Upgrade to DataFusion 40 [#657](https://github.com/apache/datafusion-comet/pull/657) (andygrove) - feat: Show user a more intuitive message when queries fall back to Spark [#656](https://github.com/apache/datafusion-comet/pull/656) (andygrove) - feat: Enable remaining Spark 3.5.1 tests [#676](https://github.com/apache/datafusion-comet/pull/676) (andygrove) - feat: Spark-4.0 widening type support [#604](https://github.com/apache/datafusion-comet/pull/604) (kazuyukitanimura) - feat: add scalar subquery pushdown to scan [#678](https://github.com/apache/datafusion-comet/pull/678) (parthchandra) **Fixed bugs:** - fix: Comet sink operator should not have children operators [#26](https://github.com/apache/datafusion-comet/pull/26) (viirya) - fix: Fix the UnionExec match branches in CometExecRule [#68](https://github.com/apache/datafusion-comet/pull/68) (wankunde) - fix: Appending null values to element array builders of StructBuilder for null row in a StructArray [#78](https://github.com/apache/datafusion-comet/pull/78) (viirya) - fix: Fix compilation error for CometBroadcastExchangeExec [#86](https://github.com/apache/datafusion-comet/pull/86) (viirya) - fix: Avoid exception caused by broadcasting empty result [#92](https://github.com/apache/datafusion-comet/pull/92) (wForget) - fix: Add num_rows when building RecordBatch [#103](https://github.com/apache/datafusion-comet/pull/103) (advancedxy) - fix: Cast string to boolean not compatible with Spark [#107](https://github.com/apache/datafusion-comet/pull/107) (erenavsarogullari) - fix: Another attempt to fix libcrypto.dylib loading issue [#112](https://github.com/apache/datafusion-comet/pull/112) (advancedxy) - fix: Fix compilation error for Spark 3.2 & 3.3 [#117](https://github.com/apache/datafusion-comet/pull/117) (sunchao) - fix: Fix corrupted AggregateMode when transforming plan parameters [#118](https://github.com/apache/datafusion-comet/pull/118) (viirya) - fix: bitwise shift with different left/right types [#135](https://github.com/apache/datafusion-comet/pull/135) (viirya) - fix: Avoid null exception in removeSubquery [#147](https://github.com/apache/datafusion-comet/pull/147) (viirya) - fix: rat check error in vscode ide [#161](https://github.com/apache/datafusion-comet/pull/161) (thexiay) - fix: Final aggregation should not bind to the input of partial aggregation [#155](https://github.com/apache/datafusion-comet/pull/155) (viirya) - fix: coalesce should return correct datatype [#168](https://github.com/apache/datafusion-comet/pull/168) (viirya) - fix: attempt to divide by zero error on decimal division [#172](https://github.com/apache/datafusion-comet/pull/172) (viirya) - fix: Aggregation without aggregation expressions should use correct result expressions [#175](https://github.com/apache/datafusion-comet/pull/175) (viirya) - fix: Comet native operator can be executed after ReusedExchange [#187](https://github.com/apache/datafusion-comet/pull/187) (viirya) - fix: Try to convert a static list into a set in Rust [#184](https://github.com/apache/datafusion-comet/pull/184) (advancedxy) - fix: Include active spiller when computing peak shuffle memory [#196](https://github.com/apache/datafusion-comet/pull/196) (sunchao) - fix: CometExecRule should handle ShuffleQueryStage and ReusedExchange [#186](https://github.com/apache/datafusion-comet/pull/186) (viirya) - fix: Use `makeCopy` to change relation in `FileSourceScanExec` [#207](https://github.com/apache/datafusion-comet/pull/207) (viirya) - fix: Remove duplicate byte array allocation for CometDictionary [#224](https://github.com/apache/datafusion-comet/pull/224) (viirya) - fix: Remove redundant data copy in columnar shuffle [#233](https://github.com/apache/datafusion-comet/pull/233) (viirya) - fix: Only maps FIXED_LEN_BYTE_ARRAY to String for uuid type [#238](https://github.com/apache/datafusion-comet/pull/238) (huaxingao) - fix: Reduce RowPartition memory allocation [#244](https://github.com/apache/datafusion-comet/pull/244) (viirya) - fix: Remove wrong calculation for Murmur3Hash for float with null input [#245](https://github.com/apache/datafusion-comet/pull/245) (advancedxy) - fix: Deallocate row addresses and size arrays after exporting [#246](https://github.com/apache/datafusion-comet/pull/246) (viirya) - fix: Fix wrong children expression order in IfExpr [#249](https://github.com/apache/datafusion-comet/pull/249) (viirya) - fix: Average expression in Comet Final should handle all null inputs from partial Spark aggregation [#261](https://github.com/apache/datafusion-comet/pull/261) (viirya) - fix: Only trigger Comet Final aggregation on Comet partial aggregation [#264](https://github.com/apache/datafusion-comet/pull/264) (viirya) - fix: incorrect result on Comet multiple column distinct count [#268](https://github.com/apache/datafusion-comet/pull/268) (viirya) - fix: Avoid using CometConf [#266](https://github.com/apache/datafusion-comet/pull/266) (snmvaughan) - fix: Fix arrow error when sorting on empty batch [#271](https://github.com/apache/datafusion-comet/pull/271) (viirya) - fix: Include license using `#` instead of using XML comment [#274](https://github.com/apache/datafusion-comet/pull/274) (snmvaughan) - fix: Comet should not translate try_sum to native sum expression [#277](https://github.com/apache/datafusion-comet/pull/277) (viirya) - fix: incorrect result with aggregate expression with filter [#284](https://github.com/apache/datafusion-comet/pull/284) (viirya) - fix: Comet should not fail on negative limit parameter [#288](https://github.com/apache/datafusion-comet/pull/288) (viirya) - fix: Comet columnar shuffle should not be on top of another Comet shuffle operator [#296](https://github.com/apache/datafusion-comet/pull/296) (viirya) - fix: Iceberg scan transition should be in front of other data source v2 [#302](https://github.com/apache/datafusion-comet/pull/302) (viirya) - fix: CometExec's outputPartitioning might not be same as Spark expects after AQE interferes [#299](https://github.com/apache/datafusion-comet/pull/299) (viirya) - fix: CometShuffleExchangeExec logical link should be correct [#324](https://github.com/apache/datafusion-comet/pull/324) (viirya) - fix: SortMergeJoin with unsupported key type should fall back to Spark [#355](https://github.com/apache/datafusion-comet/pull/355) (viirya) - fix: limit with offset should return correct results [#359](https://github.com/apache/datafusion-comet/pull/359) (viirya) - fix: Disable Comet shuffle with AQE coalesce partitions enabled [#380](https://github.com/apache/datafusion-comet/pull/380) (viirya) - fix: Unknown operator id when explain with formatted mode [#410](https://github.com/apache/datafusion-comet/pull/410) (leoluan2009) - fix: Reuse CometBroadcastExchangeExec with Spark ReuseExchangeAndSubquery rule [#441](https://github.com/apache/datafusion-comet/pull/441) (viirya) - fix: newFileScanRDD should not take constructor from custom Spark versions [#412](https://github.com/apache/datafusion-comet/pull/412) (ceppelli) - fix: fix CometNativeExec.doCanonicalize for ReusedExchangeExec [#447](https://github.com/apache/datafusion-comet/pull/447) (viirya) - fix: Enable cast string to int tests and fix compatibility issue [#453](https://github.com/apache/datafusion-comet/pull/453) (andygrove) - fix: Compute murmur3 hash with dictionary input correctly [#433](https://github.com/apache/datafusion-comet/pull/433) (advancedxy) - fix: Only delegate to DataFusion cast when we know that it is compatible with Spark [#461](https://github.com/apache/datafusion-comet/pull/461) (andygrove) - fix: `ColumnReader.loadVector` should initiate `CometDictionary` after re-import arrays [#473](https://github.com/apache/datafusion-comet/pull/473) (viirya) - fix: substring with negative indices should produce correct result [#470](https://github.com/apache/datafusion-comet/pull/470) (sonhmai) - fix: CometReader.loadVector should not overwrite dictionary ids [#476](https://github.com/apache/datafusion-comet/pull/476) (viirya) - fix: Reuse previous CometDictionary Java arrays [#489](https://github.com/apache/datafusion-comet/pull/489) (viirya) - fix: Fallback to Spark for LIKE with custom escape character [#478](https://github.com/apache/datafusion-comet/pull/478) (sujithjay) - fix: Incorrect input schema when preparing result expressions for HashAggregation [#501](https://github.com/apache/datafusion-comet/pull/501) (viirya) - fix: Input batch to ShuffleRepartitioner.insert_batch should not be larger than configured batch size [#523](https://github.com/apache/datafusion-comet/pull/523) (viirya) - fix: Fix integer overflow in date_parser [#529](https://github.com/apache/datafusion-comet/pull/529) (eejbyfeldt) - fix: null character not permitted in chr function [#513](https://github.com/apache/datafusion-comet/pull/513) (vaibhawvipul) - fix: Overflow when reading Timestamp from parquet file [#542](https://github.com/apache/datafusion-comet/pull/542) (eejbyfeldt) - fix: Re-implement some Parquet decode methods without `copy_nonoverlapping` [#558](https://github.com/apache/datafusion-comet/pull/558) (andygrove) - fix: requested character too large for encoding in chr function [#552](https://github.com/apache/datafusion-comet/pull/552) (vaibhawvipul) - fix: Running cargo build always triggers rebuild [#579](https://github.com/apache/datafusion-comet/pull/579) (eejbyfeldt) - fix: Avoid recursive call to `canonicalizePlans` [#582](https://github.com/apache/datafusion-comet/pull/582) (viirya) - fix: Return error in pre_timestamp_cast instead of panic [#543](https://github.com/apache/datafusion-comet/pull/543) (eejbyfeldt) - perf: Add criterion benchmark for xxhash64 function [#560](https://github.com/apache/datafusion-comet/pull/560) (andygrove) - fix: Fix range out of index error with a temporary workaround [#584](https://github.com/apache/datafusion-comet/pull/584) (viirya) - fix: Improve error "BroadcastExchange is not supported" [#577](https://github.com/apache/datafusion-comet/pull/577) (parthchandra) - fix: Avoid creating huge duplicate of canonicalized plans for CometNativeExec [#639](https://github.com/apache/datafusion-comet/pull/639) (viirya) - fix: Tag ignored tests that require SubqueryBroadcastExec [#647](https://github.com/apache/datafusion-comet/pull/647) (parthchandra) - fix: Optimize some functions to rewrite dictionary-encoded strings [#627](https://github.com/apache/datafusion-comet/pull/627) (vaibhawvipul) - fix: Remove nightly flag in release-nogit target in Makefile [#667](https://github.com/apache/datafusion-comet/pull/667) (andygrove) - fix: change the not exists base image apache/spark:3.4.3 to 3.4.2 [#686](https://github.com/apache/datafusion-comet/pull/686) (haoxins) - fix: Spark 4.0 SparkArithmeticException test [#688](https://github.com/apache/datafusion-comet/pull/688) (kazuyukitanimura) - fix: address failure caused by method signature change in SPARK-48791 [#693](https://github.com/apache/datafusion-comet/pull/693) (parthchandra) **Documentation updates:** - doc: Add Quickstart Comet doc section [#125](https://github.com/apache/datafusion-comet/pull/125) (comphead) - doc: Minor fix Getting started reformatting [#128](https://github.com/apache/datafusion-comet/pull/128) (comphead) - doc: Add initial doc how to expand Comet exceptions [#170](https://github.com/apache/datafusion-comet/pull/170) (comphead) - doc: Update README.md with shuffle configs [#208](https://github.com/apache/datafusion-comet/pull/208) (viirya) - doc: Update supported expressions [#237](https://github.com/apache/datafusion-comet/pull/237) (viirya) - doc: Fix a small typo in README.md [#272](https://github.com/apache/datafusion-comet/pull/272) (rz-vastdata) - doc: Update DataFusion project name and url [#300](https://github.com/apache/datafusion-comet/pull/300) (viirya) - docs: Move existing documentation into new Contributor Guide and add Getting Started section [#334](https://github.com/apache/datafusion-comet/pull/334) (andygrove) - docs: Add more content to the user guide [#347](https://github.com/apache/datafusion-comet/pull/347) (andygrove) - docs: Generate configuration guide in mvn build [#349](https://github.com/apache/datafusion-comet/pull/349) (andygrove) - docs: Add a plugin overview page to the contributors guide [#345](https://github.com/apache/datafusion-comet/pull/345) (andygrove) - doc: Fix target typo in development.md [#364](https://github.com/apache/datafusion-comet/pull/364) (jc4x4) - doc: Clean up supported JDKs in README [#366](https://github.com/apache/datafusion-comet/pull/366) (edmondop) - doc: add contributing in README.md [#382](https://github.com/apache/datafusion-comet/pull/382) (caicancai) - docs: fix the docs url of installation instructions [#393](https://github.com/apache/datafusion-comet/pull/393) (haoxins) - docs: Running ScalaTest suites from the CLI [#404](https://github.com/apache/datafusion-comet/pull/404) (edmondop) - docs: Remove spark.comet.exec.broadcast.enabled from config docs [#421](https://github.com/apache/datafusion-comet/pull/421) (andygrove) - docs: fix various sphinx warnings [#428](https://github.com/apache/datafusion-comet/pull/428) (tshauck) - doc: Add Plan Stability Testing to development guide [#432](https://github.com/apache/datafusion-comet/pull/432) (viirya) - docs: Update Spark shell command to include setting additional class path [#435](https://github.com/apache/datafusion-comet/pull/435) (andygrove) - doc: Add Tuning Guide with shuffle configs [#443](https://github.com/apache/datafusion-comet/pull/443) (viirya) - docs: Add benchmarking guide [#444](https://github.com/apache/datafusion-comet/pull/444) (andygrove) - docs: add guide to adding a new expression [#422](https://github.com/apache/datafusion-comet/pull/422) (tshauck) - docs: changes in documentation [#512](https://github.com/apache/datafusion-comet/pull/512) (SemyonSinchenko) - docs: Improve user documentation for supported operators and expressions [#520](https://github.com/apache/datafusion-comet/pull/520) (andygrove) - docs: Proposal for source release process [#556](https://github.com/apache/datafusion-comet/pull/556) (andygrove) - docs: Update benchmark results [#687](https://github.com/apache/datafusion-comet/pull/687) (andygrove) - docs: Update percentage speedups in benchmarking guide [#691](https://github.com/apache/datafusion-comet/pull/691) (andygrove) - doc: Add memory tuning section to user guide [#684](https://github.com/apache/datafusion-comet/pull/684) (viirya) **Other:** - Initial PR [#1](https://github.com/apache/datafusion-comet/pull/1) (sunchao) - build: Add Maven wrapper to the project [#13](https://github.com/apache/datafusion-comet/pull/13) (sunchao) - build: Add basic CI test pipelines [#18](https://github.com/apache/datafusion-comet/pull/18) (sunchao) - Bump com.google.protobuf:protobuf-java from 3.17.3 to 3.19.6 [#5](https://github.com/apache/datafusion-comet/pull/5) (dependabot[bot]) - build: Add PR template [#23](https://github.com/apache/datafusion-comet/pull/23) (sunchao) - build: Create ticket templates [#24](https://github.com/apache/datafusion-comet/pull/24) (comphead) - build: Re-enable Scala style checker and spotless [#21](https://github.com/apache/datafusion-comet/pull/21) (sunchao) - build: Remove license header from pull request template [#28](https://github.com/apache/datafusion-comet/pull/28) (viirya) - build: Exclude .github from apache-rat-plugin check [#32](https://github.com/apache/datafusion-comet/pull/32) (viirya) - build: Add CI for MacOS (x64 and aarch64) [#35](https://github.com/apache/datafusion-comet/pull/35) (sunchao) - fix broken link in README.md [#39](https://github.com/apache/datafusion-comet/pull/39) (nairbv) - test: Add some fuzz testing for cast operations [#16](https://github.com/apache/datafusion-comet/pull/16) (andygrove) - test: Fix CI failure on libcrypto [#41](https://github.com/apache/datafusion-comet/pull/41) (sunchao) - test: Reduce test time spent in `CometShuffleSuite` [#40](https://github.com/apache/datafusion-comet/pull/40) (sunchao) - test: Add test for RoundRobinPartitioning [#54](https://github.com/apache/datafusion-comet/pull/54) (viirya) - build: Fix potential libcrypto lib loading issue for X86 mac runners [#55](https://github.com/apache/datafusion-comet/pull/55) (advancedxy) - refactor: Remove a few duplicated occurrences [#53](https://github.com/apache/datafusion-comet/pull/53) (sunchao) - build: Fix mvn cache for containerized runners [#48](https://github.com/apache/datafusion-comet/pull/48) (advancedxy) - test: Ensure traversed operators during finding first partial aggregaion are all native [#58](https://github.com/apache/datafusion-comet/pull/58) (viirya) - build: Upgrade arrow-rs to 50.0.0 and DataFusion to 35.0.0 [#65](https://github.com/apache/datafusion-comet/pull/65) (viirya) - build: Support built with java 1.8 [#45](https://github.com/apache/datafusion-comet/pull/45) (advancedxy) - test: Add golden files for TPCDSPlanStabilitySuite [#73](https://github.com/apache/datafusion-comet/pull/73) (sunchao) - test: Add TPC-DS test results [#77](https://github.com/apache/datafusion-comet/pull/77) (sunchao) - build: Upgrade spotless version to 2.43.0 [#85](https://github.com/apache/datafusion-comet/pull/85) (viirya) - test: Expose thrown exception when executing query in CometTPCHQuerySuite [#96](https://github.com/apache/datafusion-comet/pull/96) (viirya) - test: Enable TPCDS q41 in CometTPCDSQuerySuite [#98](https://github.com/apache/datafusion-comet/pull/98) (viirya) - build: Add CI for TPCDS queries [#99](https://github.com/apache/datafusion-comet/pull/99) (viirya) - build: Add tpcds-sf-1 to license header excluded list [#108](https://github.com/apache/datafusion-comet/pull/108) (viirya) - build: Show time duration for scala test [#116](https://github.com/apache/datafusion-comet/pull/116) (advancedxy) - test: Move MacOS (x86) pipelines to post-commit [#122](https://github.com/apache/datafusion-comet/pull/122) (sunchao) - build: Upgrade DF to 36.0.0 and arrow-rs 50.0.0 [#66](https://github.com/apache/datafusion-comet/pull/66) (comphead) - test: Reduce end-to-end test time [#109](https://github.com/apache/datafusion-comet/pull/109) (sunchao) - build: Separate and speedup TPC-DS benchmark [#130](https://github.com/apache/datafusion-comet/pull/130) (advancedxy) - build: Re-enable TPCDS queries q34 and q64 in `CometTPCDSQuerySuite` [#133](https://github.com/apache/datafusion-comet/pull/133) (viirya) - build: Refine names in benchmark.yml [#132](https://github.com/apache/datafusion-comet/pull/132) (advancedxy) - build: Make the build system work out of box [#136](https://github.com/apache/datafusion-comet/pull/136) (advancedxy) - minor: Update README.md with system diagram [#148](https://github.com/apache/datafusion-comet/pull/148) (alamb) - test: Add golden files for test [#150](https://github.com/apache/datafusion-comet/pull/150) (snmvaughan) - build: Add checker for PR title [#151](https://github.com/apache/datafusion-comet/pull/151) (sunchao) - build: Support CI pipelines for Spark 3.2, 3.3 and 3.4 [#153](https://github.com/apache/datafusion-comet/pull/153) (advancedxy) - minor: Only trigger PR title checker on pull requests [#154](https://github.com/apache/datafusion-comet/pull/154) (sunchao) - chore: Fix warnings in both compiler and test environments [#164](https://github.com/apache/datafusion-comet/pull/164) (advancedxy) - build: Upload test reports and coverage [#163](https://github.com/apache/datafusion-comet/pull/163) (advancedxy) - minor: Remove unnecessary logic [#169](https://github.com/apache/datafusion-comet/pull/169) (sunchao) - minor: Make `QueryPlanSerde` warning log less confusing [#181](https://github.com/apache/datafusion-comet/pull/181) (viirya) - refactor: Skipping slicing on shuffle arrays in shuffle reader [#189](https://github.com/apache/datafusion-comet/pull/189) (viirya) - build: Run Spark SQL tests for 3.4 [#166](https://github.com/apache/datafusion-comet/pull/166) (sunchao) - build: Enforce scalafix check in CI [#203](https://github.com/apache/datafusion-comet/pull/203) (advancedxy) - test: Follow up on Spark 3.4 diff [#209](https://github.com/apache/datafusion-comet/pull/209) (sunchao) - build: Avoid confusion by using profile with clean [#215](https://github.com/apache/datafusion-comet/pull/215) (snmvaughan) - test: Add TPC-H test results [#218](https://github.com/apache/datafusion-comet/pull/218) (viirya) - build: Add CI for TPC-H queries [#220](https://github.com/apache/datafusion-comet/pull/220) (viirya) - test: Enable Comet shuffle in Spark SQL tests [#210](https://github.com/apache/datafusion-comet/pull/210) (sunchao) - test: Disable spark ui in unit test by default [#235](https://github.com/apache/datafusion-comet/pull/235) (beryllw) - chore: Replace deprecated temporal methods [#229](https://github.com/apache/datafusion-comet/pull/229) (snmvaughan) - build: Use specified branch of arrow-rs with workaround to invalid offset buffers from Java Arrow [#239](https://github.com/apache/datafusion-comet/pull/239) (viirya) - test: Enable string-to-bool cast test [#251](https://github.com/apache/datafusion-comet/pull/251) (andygrove) - test: Restore tests in CometTPCDSQuerySuite [#252](https://github.com/apache/datafusion-comet/pull/252) (viirya) - test: Enable all remaining TPCDS queries [#254](https://github.com/apache/datafusion-comet/pull/254) (viirya) - test: Enable all remaining TPCH queries [#257](https://github.com/apache/datafusion-comet/pull/257) (viirya) - chore: Remove some calls to unwrap when calling create_expr in planner.rs [#269](https://github.com/apache/datafusion-comet/pull/269) (andygrove) - chore: Fix typo in info message [#279](https://github.com/apache/datafusion-comet/pull/279) (andygrove) - chore: Fix NPE when running CometTPCHQueriesList directly [#285](https://github.com/apache/datafusion-comet/pull/285) (advancedxy) - chore: Update Comet repo description [#291](https://github.com/apache/datafusion-comet/pull/291) (viirya) - Chore: Cleanup how datafusion session config is created [#289](https://github.com/apache/datafusion-comet/pull/289) (psvri) - build: Update asf.yaml to use `@datafusion.apache.org` [#294](https://github.com/apache/datafusion-comet/pull/294) (sunchao) - chore: Remove unused functions [#301](https://github.com/apache/datafusion-comet/pull/301) (kazuyukitanimura) - chore: Ignore unused variables [#306](https://github.com/apache/datafusion-comet/pull/306) (snmvaughan) - chore: Update documentation publishing domain and path [#310](https://github.com/apache/datafusion-comet/pull/310) (andygrove) - chore: Add documentation publishing infrastructure [#314](https://github.com/apache/datafusion-comet/pull/314) (andygrove) - build: Move shim directories [#318](https://github.com/apache/datafusion-comet/pull/318) (kazuyukitanimura) - test: Suppress decimal random number tests for 3.2 and 3.3 [#319](https://github.com/apache/datafusion-comet/pull/319) (kazuyukitanimura) - chore: Add allocation source to StreamReader [#332](https://github.com/apache/datafusion-comet/pull/332) (viirya) - chore: Add more cast tests and improve test framework [#351](https://github.com/apache/datafusion-comet/pull/351) (andygrove) - chore: Implement remaining CAST tests [#356](https://github.com/apache/datafusion-comet/pull/356) (andygrove) - build: Add Spark SQL test pipeline with ANSI mode enabled [#321](https://github.com/apache/datafusion-comet/pull/321) (parthchandra) - chore: Store EXTENSION_INFO as Set[String] instead of newline-delimited String [#386](https://github.com/apache/datafusion-comet/pull/386) (andygrove) - build: Add scala-version to matrix [#396](https://github.com/apache/datafusion-comet/pull/396) (snmvaughan) - chore: Add criterion benchmarks for casting between integer types [#401](https://github.com/apache/datafusion-comet/pull/401) (andygrove) - chore: Make COMET_EXEC_BROADCAST_FORCE_ENABLED internal config [#413](https://github.com/apache/datafusion-comet/pull/413) (viirya) - chore: Rename some columnar shuffle configs for code consistently [#418](https://github.com/apache/datafusion-comet/pull/418) (leoluan2009) - chore: Remove an unused config [#430](https://github.com/apache/datafusion-comet/pull/430) (andygrove) - tests: Move random data generation methods from CometCastSuite to new DataGenerator class [#426](https://github.com/apache/datafusion-comet/pull/426) (andygrove) - test: Fix explain with exteded info comet test [#436](https://github.com/apache/datafusion-comet/pull/436) (kazuyukitanimura) - chore: Add cargo bench for shuffle writer [#438](https://github.com/apache/datafusion-comet/pull/438) (andygrove) - chore: improve fallback message when comet native shuffle is not enabled [#445](https://github.com/apache/datafusion-comet/pull/445) (andygrove) - Coverage: Add a manual test to show what Spark built in expression the DF can support directly [#331](https://github.com/apache/datafusion-comet/pull/331) (comphead) - build: Add spark-4.0 profile and shims [#407](https://github.com/apache/datafusion-comet/pull/407) (kazuyukitanimura) - build: bump spark version to 3.4.3 [#292](https://github.com/apache/datafusion-comet/pull/292) (huaxingao) - chore: Removing copying data from dictionary values into CometDictionary [#490](https://github.com/apache/datafusion-comet/pull/490) (viirya) - chore: Update README to highlight Comet benefits [#497](https://github.com/apache/datafusion-comet/pull/497) (andygrove) - test: fix ClassNotFoundException for Hive tests [#499](https://github.com/apache/datafusion-comet/pull/499) (kazuyukitanimura) - build: Enable comet tests with spark-4.0 profile [#493](https://github.com/apache/datafusion-comet/pull/493) (kazuyukitanimura) - chore: Switch to stable Rust [#505](https://github.com/apache/datafusion-comet/pull/505) (andygrove) - Minor: Generate the supported Spark builtin expression list into MD file [#455](https://github.com/apache/datafusion-comet/pull/455) (comphead) - chore: Simplify code in CometExecIterator and avoid some small overhead [#522](https://github.com/apache/datafusion-comet/pull/522) (andygrove) - chore: Upgrade spark to 4.0.0-preview1 [#526](https://github.com/apache/datafusion-comet/pull/526) (advancedxy) - chore: Add UnboundColumn to carry datatype for unbound reference [#518](https://github.com/apache/datafusion-comet/pull/518) (viirya) - chore: Remove 3.4.2.diff [#528](https://github.com/apache/datafusion-comet/pull/528) (kazuyukitanimura) - build: Switch back to official DataFusion repo and arrow-rs after Arrow Java 16 is released [#403](https://github.com/apache/datafusion-comet/pull/403) (viirya) - chore: Add CometEvalMode enum to replace string literals [#539](https://github.com/apache/datafusion-comet/pull/539) (andygrove) - chore: Create initial release process scripts for official ASF source release [#429](https://github.com/apache/datafusion-comet/pull/429) (andygrove) - build: Use DataFusion 39.0.0 release [#550](https://github.com/apache/datafusion-comet/pull/550) (viirya) - chore: disable xxhash64 by default [#548](https://github.com/apache/datafusion-comet/pull/548) (andygrove) - chore: Remove unsafe use of from_raw_parts in Parquet decoder [#549](https://github.com/apache/datafusion-comet/pull/549) (andygrove) - test: Add tests for Scalar and Inverval values for UnaryMinus [#538](https://github.com/apache/datafusion-comet/pull/538) (vaibhawvipul) - chore: Add changelog generator [#545](https://github.com/apache/datafusion-comet/pull/545) (andygrove) - chore: Remove unused hash_utils.rs [#561](https://github.com/apache/datafusion-comet/pull/561) (andygrove) - chore: Use in_list func directly [#559](https://github.com/apache/datafusion-comet/pull/559) (advancedxy) - chore: Fix most of the scala/java build warnings [#562](https://github.com/apache/datafusion-comet/pull/562) (andygrove) - chore: Upgrade to Rust 1.78 and fix UB issues in unsafe code [#546](https://github.com/apache/datafusion-comet/pull/546) (andygrove) - chore: Remove `spark.comet.xxhash64.enabled` from the config document [#586](https://github.com/apache/datafusion-comet/pull/586) (viirya) - build: Drop Spark 3.2 support [#581](https://github.com/apache/datafusion-comet/pull/581) (huaxingao) - test: Enable Spark 4.0 tests [#537](https://github.com/apache/datafusion-comet/pull/537) (kazuyukitanimura) - refactor: Remove method get_global_jclass [#580](https://github.com/apache/datafusion-comet/pull/580) (eejbyfeldt) - chore: Move some utility methods to submodules of scalar_funcs [#590](https://github.com/apache/datafusion-comet/pull/590) (advancedxy) - chore: Upgrade to Rust 1.79 [#570](https://github.com/apache/datafusion-comet/pull/570) (andygrove) - chore: Remove some calls to `unwrap` [#598](https://github.com/apache/datafusion-comet/pull/598) (andygrove) - chore: Improve JNI safety [#600](https://github.com/apache/datafusion-comet/pull/600) (andygrove) - chore: remove some unwraps from shuffle module [#601](https://github.com/apache/datafusion-comet/pull/601) (andygrove) - chore: Use proper constructor of IndexShuffleBlockResolver [#610](https://github.com/apache/datafusion-comet/pull/610) (viirya) - chore: Update benchmark results [#614](https://github.com/apache/datafusion-comet/pull/614) (andygrove) - build: Upgrade to 2.13.14 for scala-2.13 profile [#626](https://github.com/apache/datafusion-comet/pull/626) (viirya) - chore: Rename shuffle write metric [#624](https://github.com/apache/datafusion-comet/pull/624) (andygrove) - minor: replace .downcast_ref::().is_some() with .is::() [#635](https://github.com/apache/datafusion-comet/pull/635) (andygrove) - test: Add CometTPCDSQueryTestSuite [#628](https://github.com/apache/datafusion-comet/pull/628) (viirya) - chore: Convert Rust project into a workspace [#637](https://github.com/apache/datafusion-comet/pull/637) (andygrove) - chore: Add Miri workflow [#636](https://github.com/apache/datafusion-comet/pull/636) (andygrove) - test: Run optimized version of q72 derived from TPC-DS [#652](https://github.com/apache/datafusion-comet/pull/652) (viirya) - chore: Refactoring of CometError/SparkError [#655](https://github.com/apache/datafusion-comet/pull/655) (andygrove) - chore: Move `cast` to `spark-expr` crate [#654](https://github.com/apache/datafusion-comet/pull/654) (andygrove) - chore: Remove utils crate and move utils into spark-expr crate [#658](https://github.com/apache/datafusion-comet/pull/658) (andygrove) - chore: Move temporal kernels and expressions to spark-expr crate [#660](https://github.com/apache/datafusion-comet/pull/660) (andygrove) - chore: Move protobuf files to separate crate [#661](https://github.com/apache/datafusion-comet/pull/661) (andygrove) - Use IfExpr to check when input to log2 is <=0 and return null [#506](https://github.com/apache/datafusion-comet/pull/506) (PedroMDuarte) - chore: Change suffix on some expressions from Exec to Expr [#673](https://github.com/apache/datafusion-comet/pull/673) (andygrove) - chore: Fix some regressions with Spark 3.5.1 [#674](https://github.com/apache/datafusion-comet/pull/674) (andygrove) - chore: Improve fuzz testing coverage [#668](https://github.com/apache/datafusion-comet/pull/668) (andygrove) - Create Comet docker file [#675](https://github.com/apache/datafusion-comet/pull/675) (comphead) - chore: Add microbenchmarks [#671](https://github.com/apache/datafusion-comet/pull/671) (andygrove) - build: Exclude protobug generated codes from apache-rat check [#683](https://github.com/apache/datafusion-comet/pull/683) (viirya) - chore: Disable abs and signum because they return incorrect results [#695](https://github.com/apache/datafusion-comet/pull/695) (andygrove) ## Credits Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. ``` 100 Liang-Chi Hsieh 82 Andy Grove 28 advancedxy 27 Chao Sun 14 Huaxin Gao 11 KAZUYUKI TANIMURA 9 Vipul Vaibhaw 8 Parth Chandra 7 Emil Ejbyfeldt 7 Steve Vaughan 7 comphead 4 Oleks V 4 Pablo Langa 4 Trent Hauck 2 Edmondo Porcu 2 Vrishabh 2 Xin Hao 2 Xuedong Luan 1 Andrew Lamb 1 Brian Vaughan 1 Cancai Cai 1 Eren Avsarogullari 1 Holden Karau 1 JC 1 Junbo wang 1 Junfan Zhang 1 Pedro M Duarte 1 Prashant K. Sharma 1 RickestCode 1 Rohit Rastogi 1 Roman Zeyde 1 Semyon 1 Son 1 Sujith Jay Nair 1 Zhen Wang 1 ceppelli 1 dependabot[bot] 1 thexia 1 vidyasankarv 1 wankun 1 గణేష్ ``` Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. ================================================ FILE: dev/changelog/0.10.0.md ================================================ # DataFusion Comet 0.10.0 Changelog This release consists of 183 commits from 26 contributors. See credits at the end of this changelog for more information. **Fixed bugs:** - fix: [Iceberg] Fix decimal corruption [#1985](https://github.com/apache/datafusion-comet/pull/1985) (andygrove) - fix: broken link in development.md [#2024](https://github.com/apache/datafusion-comet/pull/2024) (petern48) - fix: [iceberg] Add LogicalTypeAnnotation in ParquetColumnSpec [#2000](https://github.com/apache/datafusion-comet/pull/2000) (huaxingao) - fix: hdfs read into buffer fully [#2031](https://github.com/apache/datafusion-comet/pull/2031) (parthchandra) - fix: Refactor arithmetic serde and fix correctness issues with EvalMode::TRY [#2018](https://github.com/apache/datafusion-comet/pull/2018) (andygrove) - fix: clean up [iceberg] integration APIs [#2032](https://github.com/apache/datafusion-comet/pull/2032) (huaxingao) - fix: zero Arrow Array offset before sending across FFI [#2052](https://github.com/apache/datafusion-comet/pull/2052) (mbutrovich) - fix: [iceberg] more fixes for Iceberg integration APIs. [#2078](https://github.com/apache/datafusion-comet/pull/2078) (parthchandra) - fix: Add support for StringDecode in Spark 4.0.0 [#2075](https://github.com/apache/datafusion-comet/pull/2075) (peter-toth) - fix: Avoid double free in CometUnifiedShuffleMemoryAllocator [#2122](https://github.com/apache/datafusion-comet/pull/2122) (andygrove) - fix: Remove duplicate serde code [#2098](https://github.com/apache/datafusion-comet/pull/2098) (andygrove) - fix: Improve logic for determining when an UnpackOrDeepCopy is needed [#2142](https://github.com/apache/datafusion-comet/pull/2142) (andygrove) - fix: Add CopyExec to inputs to SortMergeJoinExec [#2155](https://github.com/apache/datafusion-comet/pull/2155) (andygrove) - fix: Fix repeatedly url-decode path when reading parquet from s3 using native parquet reader [#2138](https://github.com/apache/datafusion-comet/pull/2138) (Kontinuation) - fix: [iceberg] Switch to OSS Spark and run Iceberg Spark tests in parallel [#1987](https://github.com/apache/datafusion-comet/pull/1987) (hsiang-c) - fix: [iceberg] Fall back to spark for schemas with empty structs [#2204](https://github.com/apache/datafusion-comet/pull/2204) (andygrove) - fix: Fix failing TPC-DS workflow in PR CI runs [#2207](https://github.com/apache/datafusion-comet/pull/2207) (andygrove) - fix: [iceberg] order query result deterministically [#2208](https://github.com/apache/datafusion-comet/pull/2208) (hsiang-c) - fix: use `spark.comet.batchSize` instead of `conf.arrowMaxRecordsPerBatch` for data that is coming from Java [#2196](https://github.com/apache/datafusion-comet/pull/2196) (rluvaton) - fix: if expr nullable [#2217](https://github.com/apache/datafusion-comet/pull/2217) (Asura7969) - fix: Support `auto` scan mode with Spark 4.0.0 [#1975](https://github.com/apache/datafusion-comet/pull/1975) (andygrove) - fix: Make Sha2 fallback message more user-friendly [#2213](https://github.com/apache/datafusion-comet/pull/2213) (rishvin) - fix: separate type checking for CometExchange and CometColumnarExchange [#2241](https://github.com/apache/datafusion-comet/pull/2241) (mbutrovich) - fix: Fix potential resource leak in native shuffle block reader [#2247](https://github.com/apache/datafusion-comet/pull/2247) (andygrove) - fix: Remove unreachable code in `CometScanRule` [#2252](https://github.com/apache/datafusion-comet/pull/2252) (andygrove) - fix: Fall back to `native_comet` for encrypted Parquet scans [#2250](https://github.com/apache/datafusion-comet/pull/2250) (andygrove) - fix: Fall back to `native_comet` when object store not supported by `native_iceberg_compat` [#2251](https://github.com/apache/datafusion-comet/pull/2251) (andygrove) - fix: split expr.proto file (new) [#2267](https://github.com/apache/datafusion-comet/pull/2267) (kination) - fix: handle cast to dictionary vector introduced by case when [#2044](https://github.com/apache/datafusion-comet/pull/2044) (parthchandra) - fix: Remove check for custom S3 endpoints [#2288](https://github.com/apache/datafusion-comet/pull/2288) (andygrove) - fix: implement lazy evaluation in Coalesce function [#2270](https://github.com/apache/datafusion-comet/pull/2270) (coderfender) - fix: Update benchmarking scripts [#2293](https://github.com/apache/datafusion-comet/pull/2293) (andygrove) - fix: Fix regression in NativeConfigSuite [#2299](https://github.com/apache/datafusion-comet/pull/2299) (andygrove) - fix: Validating object store configs should not throw exception [#2308](https://github.com/apache/datafusion-comet/pull/2308) (andygrove) - fix: TakeOrderedAndProjectExec is not reporting all fallback reasons [#2323](https://github.com/apache/datafusion-comet/pull/2323) (kazuyukitanimura) - fix: Fallback length function with binary input [#2349](https://github.com/apache/datafusion-comet/pull/2349) (wForget) **Performance related:** - perf: Optimize `AvgDecimalGroupsAccumulator` [#1893](https://github.com/apache/datafusion-comet/pull/1893) (leung-ming) - perf: Optimize `SumDecimalGroupsAccumulator::update_single` [#2069](https://github.com/apache/datafusion-comet/pull/2069) (leung-ming) - perf: Avoid FFI copy in `ScanExec` when reading data from exchanges [#2268](https://github.com/apache/datafusion-comet/pull/2268) (andygrove) **Implemented enhancements:** - feat: Add from_unixtime support [#1943](https://github.com/apache/datafusion-comet/pull/1943) (kazuyukitanimura) - feat: randn expression support [#2010](https://github.com/apache/datafusion-comet/pull/2010) (akupchinskiy) - feat: monotonically_increasing_id and spark_partition_id implementation [#2037](https://github.com/apache/datafusion-comet/pull/2037) (akupchinskiy) - feat: support `map_entries` [#2059](https://github.com/apache/datafusion-comet/pull/2059) (comphead) - feat: Support Array Literal [#2057](https://github.com/apache/datafusion-comet/pull/2057) (comphead) - feat: Add new trait for operator serde [#2115](https://github.com/apache/datafusion-comet/pull/2115) (andygrove) - feat: limit with offset support [#2070](https://github.com/apache/datafusion-comet/pull/2070) (akupchinskiy) - feat: Include scan implementation name in CometScan nodeName [#2141](https://github.com/apache/datafusion-comet/pull/2141) (andygrove) - feat: Add config option to log fallback reasons [#2154](https://github.com/apache/datafusion-comet/pull/2154) (andygrove) - feat: [iceberg] Enable Comet shuffle in Iceberg diff [#2205](https://github.com/apache/datafusion-comet/pull/2205) (andygrove) - feat: Improve shuffle fallback reporting [#2194](https://github.com/apache/datafusion-comet/pull/2194) (andygrove) - feat: Reset data buf of NativeBatchDecoderIterator on close [#2235](https://github.com/apache/datafusion-comet/pull/2235) (wForget) - feat: Improve fallback mechanism for ANSI mode [#2211](https://github.com/apache/datafusion-comet/pull/2211) (andygrove) - feat: Support hdfs with OpenDAL [#2244](https://github.com/apache/datafusion-comet/pull/2244) (wForget) - feat: Ignore fallback info for command execs [#2297](https://github.com/apache/datafusion-comet/pull/2297) (wForget) - feat: Improve some confusing fallback reasons [#2301](https://github.com/apache/datafusion-comet/pull/2301) (wForget) - feat: Make supported hadoop filesystem schemes configurable [#2272](https://github.com/apache/datafusion-comet/pull/2272) (wForget) - feat: [1941-Part1]: Introduce map-sort scalar function [#2262](https://github.com/apache/datafusion-comet/pull/2262) (rishvin) - feat: [iceberg] delete rows support using selection vectors [#2346](https://github.com/apache/datafusion-comet/pull/2346) (parthchandra) **Documentation updates:** - docs: Update benchmark results for 0.9.0 [#1959](https://github.com/apache/datafusion-comet/pull/1959) (andygrove) - doc: Add comment about local clippy run before submitting a pull request [#1961](https://github.com/apache/datafusion-comet/pull/1961) (akupchinskiy) - docs: Minor improvements to Spark SQL test docs [#1980](https://github.com/apache/datafusion-comet/pull/1980) (andygrove) - docs: Update Maven links for 0.9.0 release [#1988](https://github.com/apache/datafusion-comet/pull/1988) (andygrove) - docs: Documentation updates for 0.9.0 release [#1981](https://github.com/apache/datafusion-comet/pull/1981) (andygrove) - docs: Add guide showing comparison between Comet and Gluten [#2012](https://github.com/apache/datafusion-comet/pull/2012) (andygrove) - docs: Remove legacy comment in docs [#2022](https://github.com/apache/datafusion-comet/pull/2022) (andygrove) - docs: Update Gluten comparision to clarify that Velox is open-source [#2043](https://github.com/apache/datafusion-comet/pull/2043) (andygrove) - docs: Improve Gluten comparison based on feedback from the community [#2048](https://github.com/apache/datafusion-comet/pull/2048) (andygrove) - docs: added a missing export into the plan stability section [#2071](https://github.com/apache/datafusion-comet/pull/2071) (akupchinskiy) - doc: Added documentation for supported map functions [#2074](https://github.com/apache/datafusion-comet/pull/2074) (codetyri0n) - doc: Alternative way to start Spark Master to run benchmarks [#2072](https://github.com/apache/datafusion-comet/pull/2072) (comphead) - docs: Update to support try arithmetic functions [#2143](https://github.com/apache/datafusion-comet/pull/2143) (coderfender) - doc: update macos standalone spark start instructions [#2103](https://github.com/apache/datafusion-comet/pull/2103) (comphead) - docs: Update confs to bypass Iceberg Spark issues [#2166](https://github.com/apache/datafusion-comet/pull/2166) (hsiang-c) - docs: Add Roadmap [#2191](https://github.com/apache/datafusion-comet/pull/2191) (andygrove) - docs: Update installation guide for 0.9.1 [#2230](https://github.com/apache/datafusion-comet/pull/2230) (andygrove) - docs: Publish version-specific user guides [#2269](https://github.com/apache/datafusion-comet/pull/2269) (andygrove) - docs: Fix issues with publishing user guide for older Comet versions [#2284](https://github.com/apache/datafusion-comet/pull/2284) (andygrove) - docs: Move user guide docs into /user-guide/latest [#2318](https://github.com/apache/datafusion-comet/pull/2318) (andygrove) - docs: Add manual redirects from old pages that no longer exist [#2317](https://github.com/apache/datafusion-comet/pull/2317) (andygrove) - docs: Fix broken links and other Sphinx warnings [#2320](https://github.com/apache/datafusion-comet/pull/2320) (andygrove) - docs: Use `sphinx-reredirects` for redirects [#2324](https://github.com/apache/datafusion-comet/pull/2324) (andygrove) - docs: Add note about Root CA Certificate location with native scans [#2325](https://github.com/apache/datafusion-comet/pull/2325) (andygrove) - docs: Stop hard-coding Comet version in docs [#2326](https://github.com/apache/datafusion-comet/pull/2326) (andygrove) - docs: Update supported expressions and operators in user guide [#2327](https://github.com/apache/datafusion-comet/pull/2327) (andygrove) - docs: Update Iceberg docs for 0.10.0 release [#2355](https://github.com/apache/datafusion-comet/pull/2355) (hsiang-c) **Other:** - chore: Start 0.10.0 development [#1958](https://github.com/apache/datafusion-comet/pull/1958) (andygrove) - build: Fix release dockerfile [#1960](https://github.com/apache/datafusion-comet/pull/1960) (andygrove) - test: Run Iceberg Spark tests only when PR title contains [iceberg] [#1976](https://github.com/apache/datafusion-comet/pull/1976) (hsiang-c) - chore: Reuse comet allocator [#1973](https://github.com/apache/datafusion-comet/pull/1973) (EmilyMatt) - chore: update `CopyExec` with `maintains_input_order`, `supports_limit_pushdown` and `cardinality_effect` [#1979](https://github.com/apache/datafusion-comet/pull/1979) (rluvaton) - chore: extract CreateArray from QueryPlanSerde [#1991](https://github.com/apache/datafusion-comet/pull/1991) (tglanz) - chore: use DF scalar functions for StartsWith, EndsWith, Contains, DF LikeExpr [#1887](https://github.com/apache/datafusion-comet/pull/1887) (mbutrovich) - refactor: standardize div_ceil [#1999](https://github.com/apache/datafusion-comet/pull/1999) (tglanz) - Feat: support map_from_arrays [#1932](https://github.com/apache/datafusion-comet/pull/1932) (kazantsev-maksim) - chore: Implement BloomFilterMightContain as a ScalarUDFImpl [#1954](https://github.com/apache/datafusion-comet/pull/1954) (tglanz) - chore: Drop support for RightSemi and RightAnti join types [#1935](https://github.com/apache/datafusion-comet/pull/1935) (dharanad) - minor: Refactor to reduce duplicate serde code [#2011](https://github.com/apache/datafusion-comet/pull/2011) (andygrove) - chore: Introduce ANSI support for remainder operation [#1971](https://github.com/apache/datafusion-comet/pull/1971) (rishvin) - chore: Improve process for generating dynamic content into documentation [#2017](https://github.com/apache/datafusion-comet/pull/2017) (andygrove) - minor: Refactor to move some shuffle-related logic from `QueryPlanSerde` to `CometExecRule` [#2015](https://github.com/apache/datafusion-comet/pull/2015) (andygrove) - chore: Add benchmarking scripts [#2025](https://github.com/apache/datafusion-comet/pull/2025) (andygrove) - chore: Add scripts for running benchmark based on TPC-DS [#2042](https://github.com/apache/datafusion-comet/pull/2042) (andygrove) - Chore: Improve array contains test coverage [#2030](https://github.com/apache/datafusion-comet/pull/2030) (kazantsev-maksim) - fix : cast_operands_to_decimal_type_to_fix_arithmetic_overflow [#1996](https://github.com/apache/datafusion-comet/pull/1996) (coderfender) - chore: Add scripts for running benchmarks with Blaze [#2050](https://github.com/apache/datafusion-comet/pull/2050) (andygrove) - chore: migrate to DF 49.0.0 [#2040](https://github.com/apache/datafusion-comet/pull/2040) (comphead) - chore: Refactor aggregate serde to be consistent with other expression serde [#2055](https://github.com/apache/datafusion-comet/pull/2055) (andygrove) - Chore: implement string_space as ScalarUDFImpl [#2041](https://github.com/apache/datafusion-comet/pull/2041) (kazantsev-maksim) - docs : Change notes for `IntegralDivide` [#2054](https://github.com/apache/datafusion-comet/pull/2054) (coderfender) - Chore: refactor Comparison out of QueryPlanSerde [#2028](https://github.com/apache/datafusion-comet/pull/2028) (CuteChuanChuan) - chore: Use Datafusion's Sha2 and remove Comet's implementation. [#2063](https://github.com/apache/datafusion-comet/pull/2063) (rishvin) - chore: Adding dependabot [#2076](https://github.com/apache/datafusion-comet/pull/2076) (comphead) - chore: Fix clippy issues for Rust 1.89.0 [#2082](https://github.com/apache/datafusion-comet/pull/2082) (andygrove) - chore: Refactor string expression serde, part 1 [#2068](https://github.com/apache/datafusion-comet/pull/2068) (andygrove) - chore: Use `chr` function from datafusion-spark [#2080](https://github.com/apache/datafusion-comet/pull/2080) (andygrove) - minor: CometBuffer code cleanup [#2090](https://github.com/apache/datafusion-comet/pull/2090) (andygrove) - chore: Refactor string expression serde, part 2 [#2097](https://github.com/apache/datafusion-comet/pull/2097) (andygrove) - chore: create copy of fs-hdfs [#2062](https://github.com/apache/datafusion-comet/pull/2062) (parthchandra) - Chore: refactor datetime related expressions out of QueryPlanSerde [#2085](https://github.com/apache/datafusion-comet/pull/2085) (CuteChuanChuan) - chore(deps): bump actions/checkout from 3 to 4 [#2104](https://github.com/apache/datafusion-comet/pull/2104) (dependabot[bot]) - chore(deps): bump libc from 0.2.174 to 0.2.175 in /native [#2107](https://github.com/apache/datafusion-comet/pull/2107) (dependabot[bot]) - chore(deps): bump assertables from 9.8.1 to 9.8.2 in /native [#2108](https://github.com/apache/datafusion-comet/pull/2108) (dependabot[bot]) - chore: Update dependabot label [#2110](https://github.com/apache/datafusion-comet/pull/2110) (mbutrovich) - chore: Move `stringDecode()` to `CommonStringExprs` trait [#2111](https://github.com/apache/datafusion-comet/pull/2111) (peter-toth) - chore(deps): bump uuid from 0.8.2 to 1.17.0 in /native [#2106](https://github.com/apache/datafusion-comet/pull/2106) (dependabot[bot]) - chore(deps): bump actions/download-artifact from 4 to 5 [#2109](https://github.com/apache/datafusion-comet/pull/2109) (dependabot[bot]) - chore(deps): bump tokio from 1.47.0 to 1.47.1 in /native [#2112](https://github.com/apache/datafusion-comet/pull/2112) (dependabot[bot]) - chore(deps): bump actions/setup-java from 3 to 4 [#2105](https://github.com/apache/datafusion-comet/pull/2105) (dependabot[bot]) - chore(deps): bump the proto group in /native with 2 updates [#2113](https://github.com/apache/datafusion-comet/pull/2113) (dependabot[bot]) - chore: Add type parameter to `CometExpressionSerde` [#2114](https://github.com/apache/datafusion-comet/pull/2114) (peter-toth) - chore(deps): bump cc from 1.2.30 to 1.2.32 in /native [#2123](https://github.com/apache/datafusion-comet/pull/2123) (dependabot[bot]) - chore(deps): bump bindgen from 0.64.0 to 0.69.5 in /native [#2124](https://github.com/apache/datafusion-comet/pull/2124) (dependabot[bot]) - chore(deps): bump aws-credential-types from 1.2.4 to 1.2.5 in /native [#2125](https://github.com/apache/datafusion-comet/pull/2125) (dependabot[bot]) - chore(deps): bump actions/checkout from 4 to 5 [#2126](https://github.com/apache/datafusion-comet/pull/2126) (dependabot[bot]) - chore: fix `QueryPlanSerde` merge error [#2127](https://github.com/apache/datafusion-comet/pull/2127) (comphead) - chore(deps): bump slab from 0.4.10 to 0.4.11 in /native [#2128](https://github.com/apache/datafusion-comet/pull/2128) (dependabot[bot]) - fix : implement_try_eval_mode_arithmetic [#2073](https://github.com/apache/datafusion-comet/pull/2073) (coderfender) - chore: Simplify approach to avoiding memory corruption due to buffer reuse [#2156](https://github.com/apache/datafusion-comet/pull/2156) (andygrove) - chore: upgrade to DataFusion 49.0.1 [#2077](https://github.com/apache/datafusion-comet/pull/2077) (mbutrovich) - chore: CometExecRule code cleanup [#2159](https://github.com/apache/datafusion-comet/pull/2159) (andygrove) - chore: Update `CometTestBase` to stop setting the scan implementation to `native_comet` [#2176](https://github.com/apache/datafusion-comet/pull/2176) (andygrove) - trivial: remove unnecessary clone() [#2066](https://github.com/apache/datafusion-comet/pull/2066) (isimluk) - chore: Pass Spark configs to native `createPlan` [#2180](https://github.com/apache/datafusion-comet/pull/2180) (andygrove) - (feat) add support for ArrayMin scalar function [#1944](https://github.com/apache/datafusion-comet/pull/1944) (dharanad) - chore: Upgrade to 49.0.2 [#2223](https://github.com/apache/datafusion-comet/pull/2223) (comphead) - chore(deps): bump bindgen from 0.69.5 to 0.72.0 in /native [#2222](https://github.com/apache/datafusion-comet/pull/2222) (dependabot[bot]) - chore: move Round serde into object [#2237](https://github.com/apache/datafusion-comet/pull/2237) (andygrove) - chore: Improve expression fallback reporting [#2240](https://github.com/apache/datafusion-comet/pull/2240) (andygrove) - chore: Update stability suite to use `auto` scan instead of `native_comet` [#2178](https://github.com/apache/datafusion-comet/pull/2178) (andygrove) - chore: Improve documentation for `CometBatchIterator` and fix a potential issue [#2168](https://github.com/apache/datafusion-comet/pull/2168) (andygrove) - chore: Fix `array_intersect` test [#2246](https://github.com/apache/datafusion-comet/pull/2246) (comphead) - chore(deps): bump actions/checkout from 4 to 5 [#2229](https://github.com/apache/datafusion-comet/pull/2229) (dependabot[bot]) - chore(deps): bump actions/setup-java from 4 to 5 [#2225](https://github.com/apache/datafusion-comet/pull/2225) (dependabot[bot]) - chore: Introduce `strict-warning` profile for Scala [#2254](https://github.com/apache/datafusion-comet/pull/2254) (comphead) - chore: fix struct to string test for `native_iceberg_compat` [#2253](https://github.com/apache/datafusion-comet/pull/2253) (comphead) - chore: Add type parameter to CometAggregateExpressionSerde [#2249](https://github.com/apache/datafusion-comet/pull/2249) (andygrove) - Feat: Impl array flatten func [#2039](https://github.com/apache/datafusion-comet/pull/2039) (kazantsev-maksim) - Chore: Refactor serde for math expressions [#2259](https://github.com/apache/datafusion-comet/pull/2259) (kazantsev-maksim) - chore: Refactor serde for more array and struct expressions [#2257](https://github.com/apache/datafusion-comet/pull/2257) (andygrove) - chore: Refactor remaining predicate expression serde [#2265](https://github.com/apache/datafusion-comet/pull/2265) (andygrove) - chore(deps): bump procfs from 0.17.0 to 0.18.0 in /native [#2278](https://github.com/apache/datafusion-comet/pull/2278) (dependabot[bot]) - chore(deps): bump cc from 1.2.34 to 1.2.35 in /native [#2277](https://github.com/apache/datafusion-comet/pull/2277) (dependabot[bot]) - chore(deps): bump bindgen from 0.72.0 to 0.72.1 in /native [#2274](https://github.com/apache/datafusion-comet/pull/2274) (dependabot[bot]) - chore(deps): bump aws-credential-types from 1.2.5 to 1.2.6 in /native [#2275](https://github.com/apache/datafusion-comet/pull/2275) (dependabot[bot]) - minor: Remove useless ENABLE_COMET_SHUFFLE env [#2280](https://github.com/apache/datafusion-comet/pull/2280) (wForget) - chore: Refactor serde for conditional expressions [#2266](https://github.com/apache/datafusion-comet/pull/2266) (andygrove) - chore(deps): bump mimalloc from 0.1.47 to 0.1.48 in /native [#2276](https://github.com/apache/datafusion-comet/pull/2276) (dependabot[bot]) - chore: docker publish and docs build only for apache repo [#2289](https://github.com/apache/datafusion-comet/pull/2289) (wForget) - minor: Reduce misleading fallback warnings [#2283](https://github.com/apache/datafusion-comet/pull/2283) (andygrove) - chore: Refactor `Cast` serde to avoid code duplication [#2242](https://github.com/apache/datafusion-comet/pull/2242) (andygrove) - chore: Refactor `hex`/`unhex` SerDe to avoid code duplication [#2287](https://github.com/apache/datafusion-comet/pull/2287) (hsiang-c) - minor: Improve exception message for unimplemented CometVector methods [#2291](https://github.com/apache/datafusion-comet/pull/2291) (andygrove) - chore: Align sort constraints w/ `arrow-rs` [#2279](https://github.com/apache/datafusion-comet/pull/2279) (hsiang-c) - chore: Collect fallback reasons for spark sql tests [#2313](https://github.com/apache/datafusion-comet/pull/2313) (wForget) - chore: Refactor serde for named expressions `alias` and `attributeReference` [#2290](https://github.com/apache/datafusion-comet/pull/2290) (andygrove) - chore(deps): bump log4rs from 1.3.0 to 1.4.0 in /native [#2334](https://github.com/apache/datafusion-comet/pull/2334) (dependabot[bot]) - chore(deps): bump twox-hash from 2.1.1 to 2.1.2 in /native [#2335](https://github.com/apache/datafusion-comet/pull/2335) (dependabot[bot]) - chore(deps): bump actions/setup-python from 5 to 6 [#2331](https://github.com/apache/datafusion-comet/pull/2331) (dependabot[bot]) - chore(deps): bump actions/download-artifact from 4 to 5 [#2332](https://github.com/apache/datafusion-comet/pull/2332) (dependabot[bot]) - chore(deps): bump cc from 1.2.35 to 1.2.36 in /native [#2337](https://github.com/apache/datafusion-comet/pull/2337) (dependabot[bot]) - chore(deps): bump log from 0.4.27 to 0.4.28 in /native [#2333](https://github.com/apache/datafusion-comet/pull/2333) (dependabot[bot]) - build: Specify SPARK_LOCAL_HOSTNAME to fix CI failures [#2353](https://github.com/apache/datafusion-comet/pull/2353) (andygrove) - chore: [branch-0.10] Bump version to 0.10.0 [#2356](https://github.com/apache/datafusion-comet/pull/2356) (andygrove) ## Credits Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. ``` 75 Andy Grove 27 dependabot[bot] 11 Oleks V 9 Zhen Wang 7 hsiang-c 5 Artem Kupchinskiy 5 B Vadlamani 5 Kazantsev Maksim 5 Matt Butrovich 5 Parth Chandra 4 Rishab Joshi 3 Peter Toth 3 Tal Glanzman 2 Dharan Aditya 2 Huaxin Gao 2 KAZUYUKI TANIMURA 2 Leung Ming 2 Raz Luvaton 2 Yu-Chuan Hung 1 Asura7969 1 Emily Matheys 1 K.I. (Dennis) Jung 1 Kristin Cowalcijk 1 Peter Nguyen 1 codetyri0n 1 Šimon Lukašík ``` Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. ================================================ FILE: dev/changelog/0.11.0.md ================================================ # DataFusion Comet 0.11.0 Changelog This release consists of 131 commits from 15 contributors. See credits at the end of this changelog for more information. **Fixed bugs:** - fix: temporarily ignore test for hdfs file systems [#2359](https://github.com/apache/datafusion-comet/pull/2359) (parthchandra) - fix: Check reused broadcast plan in non-AQE and make setNumPartitions thread safe [#2398](https://github.com/apache/datafusion-comet/pull/2398) (wForget) - fix: correct `missingInput` for `CometHashAggregateExec` [#2409](https://github.com/apache/datafusion-comet/pull/2409) (comphead) - fix:clippy errros rust 1.9.0 update [#2419](https://github.com/apache/datafusion-comet/pull/2419) (coderfender) - fix: Avoid spark plan execution cache preventing CometBatchRDD numPartitions change [#2420](https://github.com/apache/datafusion-comet/pull/2420) (wForget) - fix: regressions in `CometToPrettyStringSuite` [#2384](https://github.com/apache/datafusion-comet/pull/2384) (hsiang-c) - fix: Byte array Literals failed on cast [#2432](https://github.com/apache/datafusion-comet/pull/2432) (comphead) - fix: Do not push down subquery filters on native_datafusion scan [#2438](https://github.com/apache/datafusion-comet/pull/2438) (wForget) - fix: Improve error handling when resolving S3 bucket region [#2440](https://github.com/apache/datafusion-comet/pull/2440) (andygrove) - fix: [iceberg] additional parquet independent api for iceberg integration [#2442](https://github.com/apache/datafusion-comet/pull/2442) (parthchandra) - fix: Specify reqwest crate features [#2446](https://github.com/apache/datafusion-comet/pull/2446) (andygrove) - fix: distributed RangePartitioning bounds calculation with native shuffle [#2258](https://github.com/apache/datafusion-comet/pull/2258) (mbutrovich) - fix: fix regression in tpcbench.py [#2512](https://github.com/apache/datafusion-comet/pull/2512) (andygrove) - fix: [iceberg] Close reader instance in ReadConf [#2510](https://github.com/apache/datafusion-comet/pull/2510) (hsiang-c) - fix: Enable plan stability tests for `auto` scan [#2516](https://github.com/apache/datafusion-comet/pull/2516) (andygrove) - fix: Capture unexpected output when retrieving JVM 17 args in Makefile [#2566](https://github.com/apache/datafusion-comet/pull/2566) (zuston) **Performance related:** - perf: New Configuration from shared conf to avoid high costs [#2402](https://github.com/apache/datafusion-comet/pull/2402) (wForget) - perf: Use DataFusion's `count_udaf` instead of `SUM(IF(expr IS NOT NULL, 1, 0))` [#2407](https://github.com/apache/datafusion-comet/pull/2407) (andygrove) - perf: Improve BroadcastExchangeExec conversion [#2417](https://github.com/apache/datafusion-comet/pull/2417) (wForget) **Implemented enhancements:** - feat: Add dynamic `enabled` and `allowIncompat` configs for all supported expressions [#2329](https://github.com/apache/datafusion-comet/pull/2329) (andygrove) - feat: feature specific tests [#2372](https://github.com/apache/datafusion-comet/pull/2372) (parthchandra) - feat: Support more date part expressions [#2316](https://github.com/apache/datafusion-comet/pull/2316) (wForget) - feat: rpad support column for second arg instead of just literal [#2099](https://github.com/apache/datafusion-comet/pull/2099) (coderfender) - feat: Support comet native log level conf [#2379](https://github.com/apache/datafusion-comet/pull/2379) (wForget) - feat: Enable WeekDay function [#2411](https://github.com/apache/datafusion-comet/pull/2411) (wForget) - feat: Add nested Array literal support [#2181](https://github.com/apache/datafusion-comet/pull/2181) (comphead) - feat:add_additional_char_support_rpad [#2436](https://github.com/apache/datafusion-comet/pull/2436) (coderfender) - feat: do not fallback to Spark for `COUNT(distinct)` [#2429](https://github.com/apache/datafusion-comet/pull/2429) (comphead) - feat: implement_ansi_eval_mode_arithmetic [#2136](https://github.com/apache/datafusion-comet/pull/2136) (coderfender) - feat: Add plan conversion statistics to extended explain info [#2412](https://github.com/apache/datafusion-comet/pull/2412) (andygrove) - feat: implement_comet_native_lpad_expr [#2102](https://github.com/apache/datafusion-comet/pull/2102) (coderfender) - feat: Add `backtrace` feature to simplify enabling native backtraces in `CometNativeException` [#2515](https://github.com/apache/datafusion-comet/pull/2515) (andygrove) - feat: Support reverse function with ArrayType input [#2481](https://github.com/apache/datafusion-comet/pull/2481) (cfmcgrady) - feat: Change default off-heap memory pool from `greedy_unified` to `fair_unified` [#2526](https://github.com/apache/datafusion-comet/pull/2526) (andygrove) - feat: Make DiskManager `max_temp_directory_size` configurable [#2479](https://github.com/apache/datafusion-comet/pull/2479) (manuzhang) - feat: Parquet Modular Encryption with Spark KMS for native readers [#2447](https://github.com/apache/datafusion-comet/pull/2447) (mbutrovich) - feat: Add support for Spark-compatible cast from integral to decimal [#2472](https://github.com/apache/datafusion-comet/pull/2472) (coderfender) - feat:Support ANSI mode integral divide [#2421](https://github.com/apache/datafusion-comet/pull/2421) (coderfender) - feat: Add config to enable running Comet in onheap mode [#2554](https://github.com/apache/datafusion-comet/pull/2554) (andygrove) - feat:support ansi mode rounding function [#2542](https://github.com/apache/datafusion-comet/pull/2542) (coderfender) - feat:support ansi mode remainder function [#2556](https://github.com/apache/datafusion-comet/pull/2556) (coderfender) - feat: Implement array-to-string cast support [#2425](https://github.com/apache/datafusion-comet/pull/2425) (cfmcgrady) - feat: Various improvements to memory pool configuration, logging, and documentation [#2538](https://github.com/apache/datafusion-comet/pull/2538) (andygrove) - feat: Enable complex types for columnar shuffle [#2573](https://github.com/apache/datafusion-comet/pull/2573) (mbutrovich) - feat: support_decimal_types_bool_cast_native_impl [#2490](https://github.com/apache/datafusion-comet/pull/2490) (coderfender) - feat: Use buf write to reduce system call on index write [#2579](https://github.com/apache/datafusion-comet/pull/2579) (zuston) **Documentation updates:** - doc: Document usage IcebergCometBatchReader.java [#2347](https://github.com/apache/datafusion-comet/pull/2347) (comphead) - docs: Add changelog for 0.10.0 release [#2361](https://github.com/apache/datafusion-comet/pull/2361) (andygrove) - docs: Fix error in docs [#2373](https://github.com/apache/datafusion-comet/pull/2373) (andygrove) - docs: Fix more comet versions in docs [#2374](https://github.com/apache/datafusion-comet/pull/2374) (andygrove) - docs: Publish 0.10.0 user guide [#2394](https://github.com/apache/datafusion-comet/pull/2394) (andygrove) - doc: macos benches doc clarifications [#2418](https://github.com/apache/datafusion-comet/pull/2418) (comphead) - docs: update configs.md after #2422 [#2428](https://github.com/apache/datafusion-comet/pull/2428) (mbutrovich) - docs: update docs and tuning guide related to native shuffle [#2487](https://github.com/apache/datafusion-comet/pull/2487) (mbutrovich) - docs: Improve EC2 benchmarking guide [#2474](https://github.com/apache/datafusion-comet/pull/2474) (andygrove) - docs: docs_update_ansi_support [#2496](https://github.com/apache/datafusion-comet/pull/2496) (coderfender) - docs:support lpad expression documentation update [#2517](https://github.com/apache/datafusion-comet/pull/2517) (coderfender) - docs: doc changes to support ANSI mode integral divide [#2570](https://github.com/apache/datafusion-comet/pull/2570) (coderfender) - docs: Split configuration guide into different sections (scan, exec, shuffle, etc) [#2568](https://github.com/apache/datafusion-comet/pull/2568) (andygrove) - docs: doc update to support ANSI mode remainder function [#2576](https://github.com/apache/datafusion-comet/pull/2576) (coderfender) - docs: Documentation updates [#2581](https://github.com/apache/datafusion-comet/pull/2581) (andygrove) **Other:** - chore(deps): bump uuid from 1.18.0 to 1.18.1 in /native [#2336](https://github.com/apache/datafusion-comet/pull/2336) (dependabot[bot]) - build: Check that all Scala test suites run in PR builds [#2304](https://github.com/apache/datafusion-comet/pull/2304) (andygrove) - chore: Start 0.11.0 development [#2365](https://github.com/apache/datafusion-comet/pull/2365) (andygrove) - chore: Split expression serde hash map into separate categories [#2322](https://github.com/apache/datafusion-comet/pull/2322) (andygrove) - chore: exclude Iceberg diffs from rat checks [#2376](https://github.com/apache/datafusion-comet/pull/2376) (hsiang-c) - chore: Refactor UnaryMinus serde [#2378](https://github.com/apache/datafusion-comet/pull/2378) (andygrove) - chore: Revert "chore: [1941-Part1]: Introduce `map_sort` scalar function (#2… [#2381](https://github.com/apache/datafusion-comet/pull/2381) (comphead) - chore: Refactor Literal serde [#2377](https://github.com/apache/datafusion-comet/pull/2377) (andygrove) - chore: Output `BaseAggregateExec` accurate unsupported names [#2383](https://github.com/apache/datafusion-comet/pull/2383) (comphead) - chore: Improve Initcap test and docs [#2387](https://github.com/apache/datafusion-comet/pull/2387) (andygrove) - build: fix build of 'hdfs-opendal' feature for MacOS [#2392](https://github.com/apache/datafusion-comet/pull/2392) (parthchandra) - chore(deps): bump cc from 1.2.36 to 1.2.37 in /native [#2399](https://github.com/apache/datafusion-comet/pull/2399) (dependabot[bot]) - chore: [iceberg] support Iceberg 1.9.1 [#2386](https://github.com/apache/datafusion-comet/pull/2386) (hsiang-c) - minor: Add deprecation notice to `datafusion-comet-spark-expr` crate [#2405](https://github.com/apache/datafusion-comet/pull/2405) (andygrove) - minor: Update benchmarking scripts to specify scan implementation [#2403](https://github.com/apache/datafusion-comet/pull/2403) (andygrove) - refactor: Scala hygiene - remove `scala.collection.JavaConverters` [#2393](https://github.com/apache/datafusion-comet/pull/2393) (hsiang-c) - chore: Improve test coverage for `count` aggregates [#2406](https://github.com/apache/datafusion-comet/pull/2406) (andygrove) - chore: upgrade to DataFusion 50.0.0, Arrow 56.1.0, Parquet 56.0.0 among others [#2286](https://github.com/apache/datafusion-comet/pull/2286) (mbutrovich) - chore: Support Spark 4.0.1 instead of 4.0.0 [#2414](https://github.com/apache/datafusion-comet/pull/2414) (andygrove) - chore: Respect native features env for cargo commands [#2296](https://github.com/apache/datafusion-comet/pull/2296) (wForget) - minor: Update TPC-DS microbenchmarks to remove "scan only" and "exec only" runs [#2396](https://github.com/apache/datafusion-comet/pull/2396) (andygrove) - minor: Add RDDScan to default value of sparkToColumnar.supportedOperatorList [#2422](https://github.com/apache/datafusion-comet/pull/2422) (wForget) - chore: new TPC-DS golden plans [#2426](https://github.com/apache/datafusion-comet/pull/2426) (mbutrovich) - chore: fix `pr_build*.yml` [#2434](https://github.com/apache/datafusion-comet/pull/2434) (comphead) - chore: Remove unused class [#2437](https://github.com/apache/datafusion-comet/pull/2437) (wForget) - chore(deps): bump cc from 1.2.37 to 1.2.38 in /native [#2439](https://github.com/apache/datafusion-comet/pull/2439) (dependabot[bot]) - chore: add validate_workflows.yml [#2441](https://github.com/apache/datafusion-comet/pull/2441) (comphead) - test: potential native broadcast failure in scenarios with ReusedExhange [#2167](https://github.com/apache/datafusion-comet/pull/2167) (akupchinskiy) - chore: Improvements of fallback info [#2450](https://github.com/apache/datafusion-comet/pull/2450) (wForget) - chore: Upgrade Apache Release Audit Tool (RAT) to 0.16.1 [#2451](https://github.com/apache/datafusion-comet/pull/2451) (andygrove) - minor: Remove reference to SortExec deadlock issue that is now resolved [#2464](https://github.com/apache/datafusion-comet/pull/2464) (andygrove) - chore: Use checked operations when growing or shrinking unified memory pool [#2455](https://github.com/apache/datafusion-comet/pull/2455) (andygrove) - minor: Improve the log message of `CometTestBase#checkCometOperators` [#2458](https://github.com/apache/datafusion-comet/pull/2458) (cfmcgrady) - minor: Skip calculating per-task memory limit when in off-heap mode [#2462](https://github.com/apache/datafusion-comet/pull/2462) (andygrove) - Chore: Used DataFusion impl of bit_get function [#2466](https://github.com/apache/datafusion-comet/pull/2466) (kazantsev-maksim) - chore(deps): bump regex from 1.11.2 to 1.11.3 in /native [#2483](https://github.com/apache/datafusion-comet/pull/2483) (dependabot[bot]) - chore: update TPS-DS plans after #2429 [#2486](https://github.com/apache/datafusion-comet/pull/2486) (mbutrovich) - chore(deps): bump thiserror from 2.0.16 to 2.0.17 in /native [#2485](https://github.com/apache/datafusion-comet/pull/2485) (dependabot[bot]) - chore(deps): bump cc from 1.2.38 to 1.2.39 in /native [#2484](https://github.com/apache/datafusion-comet/pull/2484) (dependabot[bot]) - chore: Support running specific benchmark query [#2491](https://github.com/apache/datafusion-comet/pull/2491) (comphead) - chore: Make CometColumnarToRowExec extends CometPlan [#2460](https://github.com/apache/datafusion-comet/pull/2460) (wForget) - chore: Update artifacts to 0.10.0 [#2500](https://github.com/apache/datafusion-comet/pull/2500) (comphead) - build: Stop caching libcomet in CI [#2498](https://github.com/apache/datafusion-comet/pull/2498) (andygrove) - chore: Upgrade Maven plugins [#2494](https://github.com/apache/datafusion-comet/pull/2494) (andygrove) - Chore: Used DataFusion impl of date_add and date_sub functions [#2473](https://github.com/apache/datafusion-comet/pull/2473) (kazantsev-maksim) - minor: include taskAttemptId in log messages [#2467](https://github.com/apache/datafusion-comet/pull/2467) (andygrove) - chore: Improve test assertions in plan stability suite [#2505](https://github.com/apache/datafusion-comet/pull/2505) (andygrove) - build: Add Spark 4.0 to release build script [#2514](https://github.com/apache/datafusion-comet/pull/2514) (parthchandra) - chore: Enable plan stability tests for `native_iceberg_compat` [#2519](https://github.com/apache/datafusion-comet/pull/2519) (andygrove) - chore(deps): bump parking_lot from 0.12.4 to 0.12.5 in /native [#2530](https://github.com/apache/datafusion-comet/pull/2530) (dependabot[bot]) - chore(deps): bump cc from 1.2.39 to 1.2.40 in /native [#2529](https://github.com/apache/datafusion-comet/pull/2529) (dependabot[bot]) - chore: Refactor serde for `ArrayCompact` and `ArrayFilter` [#2536](https://github.com/apache/datafusion-comet/pull/2536) (andygrove) - Chore: Fix Scala code warnings - common module [#2527](https://github.com/apache/datafusion-comet/pull/2527) (andy-hf-kwok) - chore: Refactor serde for `CheckOverflow` [#2537](https://github.com/apache/datafusion-comet/pull/2537) (andygrove) - build: Run scala tests against release build of native code [#2541](https://github.com/apache/datafusion-comet/pull/2541) (andygrove) - chore: Pass Comet configs to native `createPlan` [#2543](https://github.com/apache/datafusion-comet/pull/2543) (andygrove) - chore: Refactor serde for Length [#2547](https://github.com/apache/datafusion-comet/pull/2547) (andygrove) - chore: Include spark shim sources for spotless plugin and reformat [#2557](https://github.com/apache/datafusion-comet/pull/2557) (wForget) - chore(deps): bump opendal from 0.54.0 to 0.54.1 in /native [#2559](https://github.com/apache/datafusion-comet/pull/2559) (dependabot[bot]) - chore: Finish moving Cast serde out of QueryPlanSerde [#2550](https://github.com/apache/datafusion-comet/pull/2550) (andygrove) - chore: Use cargo-nextest in CI [#2546](https://github.com/apache/datafusion-comet/pull/2546) (andygrove) - chore: Delete unused code [#2565](https://github.com/apache/datafusion-comet/pull/2565) (zuston) - chore: Improve plan comet transformation log [#2564](https://github.com/apache/datafusion-comet/pull/2564) (wForget) - chore(deps): bump cc from 1.2.40 to 1.2.41 in /native [#2560](https://github.com/apache/datafusion-comet/pull/2560) (dependabot[bot]) - chore(deps): bump aws-credential-types from 1.2.6 to 1.2.7 in /native [#2563](https://github.com/apache/datafusion-comet/pull/2563) (dependabot[bot]) - chore: Refactor serde for RegExpReplace [#2548](https://github.com/apache/datafusion-comet/pull/2548) (andygrove) - chore: use polymorphic map builders in shuffle. [#2571](https://github.com/apache/datafusion-comet/pull/2571) (ashdnazg) - chore: Move ToPrettyString serde into shim layer [#2549](https://github.com/apache/datafusion-comet/pull/2549) (andygrove) - chore(deps): bump DataFusion dependencies to 50.2.0, refresh Cargo.lock [#2575](https://github.com/apache/datafusion-comet/pull/2575) (mbutrovich) ## Credits Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. ``` 47 Andy Grove 15 Zhen Wang 14 B Vadlamani 12 Oleks V 11 dependabot[bot] 10 Matt Butrovich 5 Parth Chandra 5 hsiang-c 3 Fu Chen 3 Junfan Zhang 2 Kazantsev Maksim 1 Artem Kupchinskiy 1 Eshed Schacham 1 Manu Zhang 1 andy-hf-kwok ``` Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. ================================================ FILE: dev/changelog/0.12.0.md ================================================ # DataFusion Comet 0.12.0 Changelog This release consists of 105 commits from 13 contributors. See credits at the end of this changelog for more information. **Fixed bugs:** - fix: Fix `None.get` in `stringDecode` when `bin` child cannot be converted [#2606](https://github.com/apache/datafusion-comet/pull/2606) (cfmcgrady) - fix: Update FuzzDataGenerator to produce dictionary-encoded string arrays & fix bugs that this exposes [#2635](https://github.com/apache/datafusion-comet/pull/2635) (andygrove) - fix: Fallback to Spark for lpad/rpad for unsupported arguments & fix negative length handling [#2630](https://github.com/apache/datafusion-comet/pull/2630) (andygrove) - fix: Mark SortOrder with floating-point as incompatible [#2650](https://github.com/apache/datafusion-comet/pull/2650) (andygrove) - fix: Fall back to Spark for `trunc` / `date_trunc` functions when format string is unsupported, or is not a literal value [#2634](https://github.com/apache/datafusion-comet/pull/2634) (andygrove) - fix: [native_datafusion] only pass single partition of PartitionedFiles into DataSourceExec [#2675](https://github.com/apache/datafusion-comet/pull/2675) (mbutrovich) - fix: Fix subcommands options in fuzz-testing [#2684](https://github.com/apache/datafusion-comet/pull/2684) (manuzhang) - fix: Do not replace SMJ with HJ for `LeftSemi` [#2687](https://github.com/apache/datafusion-comet/pull/2687) (comphead) - fix: Apply spotless on Iceberg 1.8.1 diff [iceberg] [#2700](https://github.com/apache/datafusion-comet/pull/2700) (hsiang-c) - fix: Fix generate-user-guide-reference-docs failure when mvn command is not executed at root [#2691](https://github.com/apache/datafusion-comet/pull/2691) (manuzhang) - fix: Fix missing SortOrder fallback reason in range partitioning [#2716](https://github.com/apache/datafusion-comet/pull/2716) (andygrove) - fix: CometLiteral class cast exception with arrays [#2718](https://github.com/apache/datafusion-comet/pull/2718) (andygrove) - fix: NormalizeNaNAndZero::children() returns child's child [#2732](https://github.com/apache/datafusion-comet/pull/2732) (mbutrovich) - fix: checkSparkMaybeThrows should compare Spark and Comet results in success case [#2728](https://github.com/apache/datafusion-comet/pull/2728) (andygrove) - fix: Mark `WindowsExec` as incompatible [#2748](https://github.com/apache/datafusion-comet/pull/2748) (andygrove) - fix: Add strict floating point mode and fallback to Spark for min/max/sort on floating point inputs when enabled [#2747](https://github.com/apache/datafusion-comet/pull/2747) (andygrove) - fix: Implement producedAttributes for CometWindowExec [#2789](https://github.com/apache/datafusion-comet/pull/2789) (rahulbabarwal89) - fix: Pass all Comet configs to native plan [#2801](https://github.com/apache/datafusion-comet/pull/2801) (andygrove) **Implemented enhancements:** - feat: Add option to write benchmark results to file [#2640](https://github.com/apache/datafusion-comet/pull/2640) (andygrove) - feat: Implement metrics for iceberg compat [#2615](https://github.com/apache/datafusion-comet/pull/2615) (EmilyMatt) - feat: Define function signatures in CometFuzz [#2614](https://github.com/apache/datafusion-comet/pull/2614) (andygrove) - feat: cherry-pick UUID conversion logic from #2528 [#2648](https://github.com/apache/datafusion-comet/pull/2648) (mbutrovich) - feat: support `concat` for strings [#2604](https://github.com/apache/datafusion-comet/pull/2604) (comphead) - feat: Add support for `abs` [#2689](https://github.com/apache/datafusion-comet/pull/2689) (andygrove) - feat: Support variadic function in CometFuzz [#2682](https://github.com/apache/datafusion-comet/pull/2682) (manuzhang) - feat: CometExecRule refactor: Unify CometNativeExec creation with Serde in CometOperatorSerde trait [#2768](https://github.com/apache/datafusion-comet/pull/2768) (andygrove) - feat: support cot [#2755](https://github.com/apache/datafusion-comet/pull/2755) (psvri) - feat: Add bash script to build and run fuzz testing [#2686](https://github.com/apache/datafusion-comet/pull/2686) (manuzhang) - feat: Add getSupportLevel to CometAggregateExpressionSerde trait [#2777](https://github.com/apache/datafusion-comet/pull/2777) (andygrove) - feat: Add CI check to ensure generated docs are in sync with code [#2779](https://github.com/apache/datafusion-comet/pull/2779) (andygrove) - feat: Add prettier enforcement [#2783](https://github.com/apache/datafusion-comet/pull/2783) (andygrove) - feat: hyperbolic trig functions [#2784](https://github.com/apache/datafusion-comet/pull/2784) (psvri) - feat: [iceberg] Native scan by serializing FileScanTasks to iceberg-rust [#2528](https://github.com/apache/datafusion-comet/pull/2528) (mbutrovich) **Documentation updates:** - docs: Add changelog for 0.11.0 release [#2585](https://github.com/apache/datafusion-comet/pull/2585) (mbutrovich) - docs: Improve documentation layout [#2587](https://github.com/apache/datafusion-comet/pull/2587) (andygrove) - docs: Publish 0.11.0 user guide [#2589](https://github.com/apache/datafusion-comet/pull/2589) (andygrove) - docs: Put Comet logo in top nav bar, respect light/dark mode [#2591](https://github.com/apache/datafusion-comet/pull/2591) (andygrove) - docs: Improve main landing page [#2593](https://github.com/apache/datafusion-comet/pull/2593) (andygrove) - docs: Improve site navigation [#2597](https://github.com/apache/datafusion-comet/pull/2597) (andygrove) - docs: Update benchmark results [#2596](https://github.com/apache/datafusion-comet/pull/2596) (andygrove) - docs: Upgrade pydata-sphinx-theme to 0.16.1 [#2602](https://github.com/apache/datafusion-comet/pull/2602) (andygrove) - docs: Fix redirect [#2603](https://github.com/apache/datafusion-comet/pull/2603) (andygrove) - docs: Fix broken image link [#2613](https://github.com/apache/datafusion-comet/pull/2613) (andygrove) - docs: Add FFI docs to contributor guide [#2668](https://github.com/apache/datafusion-comet/pull/2668) (andygrove) - docs: Various documentation updates [#2674](https://github.com/apache/datafusion-comet/pull/2674) (andygrove) - docs: Add supported SortOrder expressions and fix a typo [#2694](https://github.com/apache/datafusion-comet/pull/2694) (andygrove) - docs: Minor docs update for running Spark SQL tests [#2712](https://github.com/apache/datafusion-comet/pull/2712) (andygrove) - docs: Update contributor guide for adding a new expression [#2704](https://github.com/apache/datafusion-comet/pull/2704) (andygrove) - docs: Documentation updates for `LocalTableScan` and `WindowExec` [#2742](https://github.com/apache/datafusion-comet/pull/2742) (andygrove) - docs: Typo fix [#2752](https://github.com/apache/datafusion-comet/pull/2752) (wForget) - docs: Categorize some configs as `testing` and add notes about known time zone issues [#2740](https://github.com/apache/datafusion-comet/pull/2740) (andygrove) - docs: Run prettier on all markdown files [#2782](https://github.com/apache/datafusion-comet/pull/2782) (andygrove) - docs: Ignore prettier formatting for generated tables [#2790](https://github.com/apache/datafusion-comet/pull/2790) (andygrove) - docs: Add new section to contributor guide, explaining how to add a new operator [#2758](https://github.com/apache/datafusion-comet/pull/2758) (andygrove) **Other:** - chore: Start 0.12.0 development [#2584](https://github.com/apache/datafusion-comet/pull/2584) (mbutrovich) - chore: Bump Spark from 3.5.6 to 3.5.7 [#2574](https://github.com/apache/datafusion-comet/pull/2574) (cfmcgrady) - chore(deps): bump parquet from 56.0.0 to 56.2.0 in /native [#2608](https://github.com/apache/datafusion-comet/pull/2608) (dependabot[bot]) - chore(deps): bump tikv-jemallocator from 0.6.0 to 0.6.1 in /native [#2609](https://github.com/apache/datafusion-comet/pull/2609) (dependabot[bot]) - chore(deps): bump tikv-jemalloc-ctl from 0.6.0 to 0.6.1 in /native [#2610](https://github.com/apache/datafusion-comet/pull/2610) (dependabot[bot]) - tests: FuzzDataGenerator instead of Parquet-specific generator [#2616](https://github.com/apache/datafusion-comet/pull/2616) (mbutrovich) - chore: Simplify on-heap memory configuration [#2599](https://github.com/apache/datafusion-comet/pull/2599) (andygrove) - Feat: Add sha1 function impl [#2471](https://github.com/apache/datafusion-comet/pull/2471) (kazantsev-maksim) - chore: Refactor Parquet/DataFrame fuzz data generators [#2629](https://github.com/apache/datafusion-comet/pull/2629) (andygrove) - chore: Remove needless from_raw calls [#2638](https://github.com/apache/datafusion-comet/pull/2638) (EmilyMatt) - chore: support DataFusion 50.3.0 [#2605](https://github.com/apache/datafusion-comet/pull/2605) (comphead) - chore(deps): bump actions/upload-artifact from 4 to 5 [#2654](https://github.com/apache/datafusion-comet/pull/2654) (dependabot[bot]) - chore(deps): bump cc from 1.2.42 to 1.2.43 in /native [#2653](https://github.com/apache/datafusion-comet/pull/2653) (dependabot[bot]) - chore(deps): bump actions/download-artifact from 5 to 6 [#2652](https://github.com/apache/datafusion-comet/pull/2652) (dependabot[bot]) - chore: extract comparison into separate tool [#2632](https://github.com/apache/datafusion-comet/pull/2632) (comphead) - chore: Various improvements to `checkSparkAnswer*` methods in `CometTestBase` [#2656](https://github.com/apache/datafusion-comet/pull/2656) (andygrove) - chore: Remove code for unpacking dictionaries prior to FilterExec [#2659](https://github.com/apache/datafusion-comet/pull/2659) (andygrove) - chore: display schema for datasets being compared [#2665](https://github.com/apache/datafusion-comet/pull/2665) (comphead) - chore: Remove `CopyExec` [#2663](https://github.com/apache/datafusion-comet/pull/2663) (andygrove) - chore: Add extended explain plans to stability suite [#2669](https://github.com/apache/datafusion-comet/pull/2669) (andygrove) - chore(deps): bump aws-config from 1.8.8 to 1.8.10 in /native [#2677](https://github.com/apache/datafusion-comet/pull/2677) (dependabot[bot]) - chore(deps): bump cc from 1.2.43 to 1.2.44 in /native [#2678](https://github.com/apache/datafusion-comet/pull/2678) (dependabot[bot]) - chore: `tpcbench` output `explain` just once and formatted [#2679](https://github.com/apache/datafusion-comet/pull/2679) (comphead) - chore: Add tolerance for `ComparisonTool` [#2699](https://github.com/apache/datafusion-comet/pull/2699) (comphead) - chore: Expand test coverage for `CometWindowsExec` [#2711](https://github.com/apache/datafusion-comet/pull/2711) (comphead) - chore: generate Float/Double NaN [#2695](https://github.com/apache/datafusion-comet/pull/2695) (hsiang-c) - minor: Combine two CI workflows for Spark SQL tests [#2727](https://github.com/apache/datafusion-comet/pull/2727) (andygrove) - chore: Improve framework for specifying that configs can be set with env vars [#2722](https://github.com/apache/datafusion-comet/pull/2722) (andygrove) - chore: Rename `COMET_EXPLAIN_VERBOSE_ENABLED` to `COMET_EXTENDED_EXPLAIN_FORMAT` and change default [#2644](https://github.com/apache/datafusion-comet/pull/2644) (andygrove) - chore: Fallback to Spark for windows functions [#2726](https://github.com/apache/datafusion-comet/pull/2726) (comphead) - chore: Refactor operator serde - part 1 [#2738](https://github.com/apache/datafusion-comet/pull/2738) (andygrove) - Feat: Add CometLocalTableScanExec operator [#2735](https://github.com/apache/datafusion-comet/pull/2735) (kazantsev-maksim) - chore(deps): bump cc from 1.2.44 to 1.2.45 in /native [#2750](https://github.com/apache/datafusion-comet/pull/2750) (dependabot[bot]) - chore(deps): bump aws-credential-types from 1.2.8 to 1.2.9 in /native [#2751](https://github.com/apache/datafusion-comet/pull/2751) (dependabot[bot]) - chore: Operator serde refactor part 2 [#2741](https://github.com/apache/datafusion-comet/pull/2741) (andygrove) - chore: Fallback to Spark for `array_reverse` for `array` [#2759](https://github.com/apache/datafusion-comet/pull/2759) (comphead) - chore: [iceberg] test iceberg 1.10.0 [#2709](https://github.com/apache/datafusion-comet/pull/2709) (manuzhang) - chore: Add `docs/comet-*` to rat exclude list [#2762](https://github.com/apache/datafusion-comet/pull/2762) (manuzhang) - Chore: Refactor static invoke exprs [#2671](https://github.com/apache/datafusion-comet/pull/2671) (kazantsev-maksim) - minor: Small refactor for consistent serde for hash aggregate [#2764](https://github.com/apache/datafusion-comet/pull/2764) (andygrove) - minor: Move `operator2Proto` to `CometExecRule` [#2767](https://github.com/apache/datafusion-comet/pull/2767) (andygrove) - chore: various refactoring changes for iceberg [iceberg] [#2680](https://github.com/apache/datafusion-comet/pull/2680) (parthchandra) - chore: Refactor CometExecRule handling of sink operators [#2771](https://github.com/apache/datafusion-comet/pull/2771) (andygrove) - minor: Refactor to move window-specific code from `QueryPlanSerde` to `CometWindowExec` [#2780](https://github.com/apache/datafusion-comet/pull/2780) (andygrove) - chore: Remove many references to `COMET_EXPR_ALLOW_INCOMPATIBLE` [#2775](https://github.com/apache/datafusion-comet/pull/2775) (andygrove) - chore: Remove COMET_EXPR_ALLOW_INCOMPATIBLE config [#2786](https://github.com/apache/datafusion-comet/pull/2786) (andygrove) - chore: check `missingInput` for Comet plan nodes [#2795](https://github.com/apache/datafusion-comet/pull/2795) (comphead) - chore: Finish refactoring expression serde out of `QueryPlanSerde` [#2791](https://github.com/apache/datafusion-comet/pull/2791) (andygrove) - chore: Update docs to fix CI after #2784 [#2799](https://github.com/apache/datafusion-comet/pull/2799) (mbutrovich) - chore: Update q79 golden plan for Spark 4.0 after #2795 [#2800](https://github.com/apache/datafusion-comet/pull/2800) (mbutrovich) - Fix: Fix null handling in CometVector implementations [#2643](https://github.com/apache/datafusion-comet/pull/2643) (cfmcgrady) ## Credits Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. ``` 54 Andy Grove 11 Oleks V 10 dependabot[bot] 9 Matt Butrovich 6 Manu Zhang 3 Fu Chen 3 Kazantsev Maksim 2 Emily Matheys 2 Vrishabh 2 hsiang-c 1 Parth Chandra 1 Zhen Wang 1 rahulbabarwal89 ``` Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. ================================================ FILE: dev/changelog/0.13.0.md ================================================ # DataFusion Comet 0.13.0 Changelog This release consists of 169 commits from 15 contributors. See credits at the end of this changelog for more information. **Fixed bugs:** - fix: NativeScan count assert firing for no reason [#2850](https://github.com/apache/datafusion-comet/pull/2850) (EmilyMatt) - fix: Correct link to tracing guide in CometConf [#2866](https://github.com/apache/datafusion-comet/pull/2866) (manuzhang) - fix: Fall back to Spark for MakeDecimal with unsupported input type [#2815](https://github.com/apache/datafusion-comet/pull/2815) (andygrove) - fix: Normalize s3 paths for PME key retriever [#2874](https://github.com/apache/datafusion-comet/pull/2874) (mbutrovich) - fix: modify CometNativeScan to generate the file partitions without instantiating RDD [#2891](https://github.com/apache/datafusion-comet/pull/2891) (mbutrovich) - fix: Modulus on decimal data type mismatch [#2922](https://github.com/apache/datafusion-comet/pull/2922) (andygrove) - fix: [iceberg] Mark nativeIcebergScanMetadata @transient [#2930](https://github.com/apache/datafusion-comet/pull/2930) (mbutrovich) - fix: enable cast tests for Spark 4.0 [#2919](https://github.com/apache/datafusion-comet/pull/2919) (manuzhang) - fix: Remove fallback for maps containing complex types [#2943](https://github.com/apache/datafusion-comet/pull/2943) (andygrove) - fix: CometShuffleManager hang by deferring SparkEnv access [#3002](https://github.com/apache/datafusion-comet/pull/3002) (Shekharrajak) - fix: format decimal to string when casting to short [#2916](https://github.com/apache/datafusion-comet/pull/2916) (manuzhang) - fix: [iceberg] reduce granularity of metrics updates in IcebergFileStream [#3050](https://github.com/apache/datafusion-comet/pull/3050) (mbutrovich) - fix: native shuffle now reports spill metrics correctly [#3197](https://github.com/apache/datafusion-comet/pull/3197) (andygrove) - fix: Prevent native write when input is not Arrow format [#3227](https://github.com/apache/datafusion-comet/pull/3227) (andygrove) - fix: Add JDK to Docker image for release build [#3262](https://github.com/apache/datafusion-comet/pull/3262) (hsiang-c) **Performance related:** - perf: [iceberg] Deduplicate serialized metadata for Iceberg native scan [#2933](https://github.com/apache/datafusion-comet/pull/2933) (mbutrovich) - perf: Use await instead of block_on in native shuffle writer [#2937](https://github.com/apache/datafusion-comet/pull/2937) (mbutrovich) - perf: refactor executePlan to try to avoid constantly entering Tokio runtime [#2938](https://github.com/apache/datafusion-comet/pull/2938) (mbutrovich) - perf: Optimize lpad/rpad to remove unnecessary memory allocations per element [#2963](https://github.com/apache/datafusion-comet/pull/2963) (andygrove) - perf: Improve performance of normalize_nan [#2999](https://github.com/apache/datafusion-comet/pull/2999) (andygrove) - perf: Improve string expression microbenchmarks [#3012](https://github.com/apache/datafusion-comet/pull/3012) (andygrove) - perf: Improve date/time microbenchmarks to avoid redundant/duplicate benchmarks [#3020](https://github.com/apache/datafusion-comet/pull/3020) (andygrove) - perf: Improve aggregate expression microbenchmarks [#3021](https://github.com/apache/datafusion-comet/pull/3021) (andygrove) - perf: Improve conditional expression microbenchmarks [#3024](https://github.com/apache/datafusion-comet/pull/3024) (andygrove) - perf: Improve performance of date truncate [#2997](https://github.com/apache/datafusion-comet/pull/2997) (andygrove) - perf: Add microbenchmark for comparison expressions [#3026](https://github.com/apache/datafusion-comet/pull/3026) (andygrove) - perf: Implement more microbenchmarks for cast expressions [#3031](https://github.com/apache/datafusion-comet/pull/3031) (andygrove) - perf: Add microbenchmark for hash expressions [#3028](https://github.com/apache/datafusion-comet/pull/3028) (andygrove) - perf: Improve performance of CAST from string to int [#3017](https://github.com/apache/datafusion-comet/pull/3017) (coderfender) - perf: Improve criterion benchmarks for cast string to int [#3049](https://github.com/apache/datafusion-comet/pull/3049) (andygrove) - perf: Additional optimizations for cast from string to int [#3048](https://github.com/apache/datafusion-comet/pull/3048) (andygrove) - perf: set DataFusion session context's target_partitions to match Spark's spark.task.cpus [#3062](https://github.com/apache/datafusion-comet/pull/3062) (mbutrovich) - perf: don't busy-poll Tokio stream for plans without CometScan [#3063](https://github.com/apache/datafusion-comet/pull/3063) (mbutrovich) - perf: minor optimizations in `process_sorted_row_partition` [#3059](https://github.com/apache/datafusion-comet/pull/3059) (andygrove) - perf: optimize complex-type hash implementations [#3140](https://github.com/apache/datafusion-comet/pull/3140) (mbutrovich) - perf: [iceberg] Remove IcebergFileStream, use iceberg-rust's parallelization, bump iceberg-rust to latest, cache SchemaAdapter [#3051](https://github.com/apache/datafusion-comet/pull/3051) (mbutrovich) - perf: [iceberg] reduce nativeIcebergScanMetadata serialization points [#3243](https://github.com/apache/datafusion-comet/pull/3243) (mbutrovich) - perf: reduce GC pressure in protobuf serialization [#3242](https://github.com/apache/datafusion-comet/pull/3242) (andygrove) - perf: cache serialized query plans to avoid per-partition serialization [#3246](https://github.com/apache/datafusion-comet/pull/3246) (andygrove) - perf: [iceberg] Use protobuf instead of JSON to serialize Iceberg partition values [#3247](https://github.com/apache/datafusion-comet/pull/3247) (parthchandra) **Implemented enhancements:** - feat: Add experimental support for native Parquet writes [#2812](https://github.com/apache/datafusion-comet/pull/2812) (andygrove) - feat: Partially implement file commit protocol for native Parquet writes [#2828](https://github.com/apache/datafusion-comet/pull/2828) (andygrove) - feat: CometNativeWriteExec support with native scan as a child [#2839](https://github.com/apache/datafusion-comet/pull/2839) (mbutrovich) - feat: Add support for `explode` and `explode_outer` for array inputs [#2836](https://github.com/apache/datafusion-comet/pull/2836) (andygrove) - feat: Support ANSI mode SUM (Decimal types) [#2826](https://github.com/apache/datafusion-comet/pull/2826) (coderfender) - feat: Add expression registry to native planner [#2851](https://github.com/apache/datafusion-comet/pull/2851) (andygrove) - feat: Implement native operator registry [#2875](https://github.com/apache/datafusion-comet/pull/2875) (andygrove) - feat: Improve fallback reporting for `native_datafusion` scan [#2879](https://github.com/apache/datafusion-comet/pull/2879) (andygrove) - feat: Enable bucket pruning with native_datafusion scans [#2888](https://github.com/apache/datafusion-comet/pull/2888) (mbutrovich) - feat: support_ansi-mode_aggregated_benchmarking [#2901](https://github.com/apache/datafusion-comet/pull/2901) (coderfender) - feat: [iceberg] REST catalog support for CometNativeIcebergScan [#2895](https://github.com/apache/datafusion-comet/pull/2895) (mbutrovich) - feat: [iceberg] Support session token in Iceberg Native scan [#2913](https://github.com/apache/datafusion-comet/pull/2913) (hsiang-c) - feat: Make shuffle writer buffer size configurable [#2899](https://github.com/apache/datafusion-comet/pull/2899) (andygrove) - feat: Add partial support for `from_json` [#2934](https://github.com/apache/datafusion-comet/pull/2934) (andygrove) - feat: Create benchmarks comet cast [#2932](https://github.com/apache/datafusion-comet/pull/2932) (coderfender) - feat: Support string decimal cast [#2925](https://github.com/apache/datafusion-comet/pull/2925) (coderfender) - feat: Remove unnecessary transition for native writes [#2960](https://github.com/apache/datafusion-comet/pull/2960) (comphead) - feat: Initial implementation of size for array inputs [#2862](https://github.com/apache/datafusion-comet/pull/2862) (andygrove) - feat: Support ANSI mode sum expr (int inputs) [#2600](https://github.com/apache/datafusion-comet/pull/2600) (coderfender) - feat: Support casting string float types [#2835](https://github.com/apache/datafusion-comet/pull/2835) (coderfender) - feat: Support ANSI mode avg expr (int inputs) [#2817](https://github.com/apache/datafusion-comet/pull/2817) (coderfender) - feat: Add support for remote Parquet HDFS writer with openDAL [#2929](https://github.com/apache/datafusion-comet/pull/2929) (comphead) - feat: Expand `murmur3` hash support to complex types [#3077](https://github.com/apache/datafusion-comet/pull/3077) (andygrove) - feat: Comet Writer should respect object store settings [#3042](https://github.com/apache/datafusion-comet/pull/3042) (comphead) - feat: add support for unix_date expression [#3141](https://github.com/apache/datafusion-comet/pull/3141) (andygrove) - feat: add partial support for date_format expression [#3201](https://github.com/apache/datafusion-comet/pull/3201) (andygrove) - feat: add complex type support to native Parquet writer [#3214](https://github.com/apache/datafusion-comet/pull/3214) (andygrove) - feat: implement framework to support multiple pyspark benchmarks [#3080](https://github.com/apache/datafusion-comet/pull/3080) (andygrove) - feat: add support for datediff expression [#3145](https://github.com/apache/datafusion-comet/pull/3145) (andygrove) - feat: Add support for `unix_timestamp` function [#2936](https://github.com/apache/datafusion-comet/pull/2936) (andygrove) - feat: add support for last_day expression [#3143](https://github.com/apache/datafusion-comet/pull/3143) (andygrove) - feat: Support left expression [#3206](https://github.com/apache/datafusion-comet/pull/3206) (Shekharrajak) - feat: Add support for round-robin partitioning in native shuffle [#3076](https://github.com/apache/datafusion-comet/pull/3076) (andygrove) - feat: Native columnar to row conversion (Phase 1) [#3221](https://github.com/apache/datafusion-comet/pull/3221) (andygrove) **Documentation updates:** - docs: add documentation for fully-native Iceberg scans [#2868](https://github.com/apache/datafusion-comet/pull/2868) (mbutrovich) - docs: Add documentation to contributor guide explaining native + JVM shuffle implementation [#3055](https://github.com/apache/datafusion-comet/pull/3055) (andygrove) - docs: add guidance on disabling constant folding for literal tests [#3200](https://github.com/apache/datafusion-comet/pull/3200) (andygrove) - docs: Add common pitfalls and improve PR checklist in development guide [#3231](https://github.com/apache/datafusion-comet/pull/3231) (andygrove) - docs: various documentation updates in preparation for next release [#3254](https://github.com/apache/datafusion-comet/pull/3254) (andygrove) - docs: Stop generating dynamic docs content in build [#3212](https://github.com/apache/datafusion-comet/pull/3212) (andygrove) - docs: document datetime rebasing and V2 API limitations for DataFusion-based scans [#3259](https://github.com/apache/datafusion-comet/pull/3259) (andygrove) - docs: Mark native_comet scan as deprecated [#3274](https://github.com/apache/datafusion-comet/pull/3274) (andygrove) **Other:** - chore: Add 0.12.0 changelog [#2811](https://github.com/apache/datafusion-comet/pull/2811) (andygrove) - chore: Prepare for 0.13.0 development [#2809](https://github.com/apache/datafusion-comet/pull/2809) (andygrove) - minor: Add microbenchmark for integer sum with grouping [#2805](https://github.com/apache/datafusion-comet/pull/2805) (andygrove) - test: extract conditional expression tests (`if`, `case_when` and `coalesce`) [#2807](https://github.com/apache/datafusion-comet/pull/2807) (rluvaton) - build: Disable caching for macOS PR builds [#2816](https://github.com/apache/datafusion-comet/pull/2816) (andygrove) - chore(deps): bump actions/checkout from 5 to 6 [#2818](https://github.com/apache/datafusion-comet/pull/2818) (dependabot[bot]) - chore(deps): bump object_store_opendal from 0.54.1 to 0.55.0 in /native [#2819](https://github.com/apache/datafusion-comet/pull/2819) (dependabot[bot]) - chore(deps): bump cc from 1.2.46 to 1.2.47 in /native [#2822](https://github.com/apache/datafusion-comet/pull/2822) (dependabot[bot]) - chore(deps): bump opendal from 0.54.1 to 0.55.0 in /native [#2821](https://github.com/apache/datafusion-comet/pull/2821) (dependabot[bot]) - chore: update `Iceberg` install docs [#2824](https://github.com/apache/datafusion-comet/pull/2824) (comphead) - chore(deps): bump cc from 1.2.47 to 1.2.48 in /native [#2833](https://github.com/apache/datafusion-comet/pull/2833) (dependabot[bot]) - chore(deps): bump the proto group in /native with 2 updates [#2832](https://github.com/apache/datafusion-comet/pull/2832) (dependabot[bot]) - minor: Clean up shuffle transformation code in `CometExecRule` [#2840](https://github.com/apache/datafusion-comet/pull/2840) (andygrove) - chore: fix broken link to Apache DataFusion Comet Overview in README [#2846](https://github.com/apache/datafusion-comet/pull/2846) (onestn) - chore: Refactor some of the scan and sink handling in `CometExecRule` to reduce duplicate code [#2844](https://github.com/apache/datafusion-comet/pull/2844) (andygrove) - deps: bump lz4_flex, downgrade prost from yanked version [#2847](https://github.com/apache/datafusion-comet/pull/2847) (mbutrovich) - minor: Move shuffle logic from `CometExecRule` to `CometShuffleExchangeExec` serde implementation [#2853](https://github.com/apache/datafusion-comet/pull/2853) (andygrove) - chore: remove coverage file auto generator [#2854](https://github.com/apache/datafusion-comet/pull/2854) (comphead) - chore(deps): bump cc from 1.2.48 to 1.2.49 in /native [#2858](https://github.com/apache/datafusion-comet/pull/2858) (dependabot[bot]) - chore: Refactor `CometExecRule` handling of `BroadcastHashJoin` and fix fallback reporting [#2856](https://github.com/apache/datafusion-comet/pull/2856) (andygrove) - chore: update actions/checkout from v4 to v6 in setup-iceberg and set… [#2857](https://github.com/apache/datafusion-comet/pull/2857) (bjornjorgensen) - minor: Small refactor in `CometExecRule` to remove confusing code and fix more fallback reporting [#2860](https://github.com/apache/datafusion-comet/pull/2860) (andygrove) - chore: Add unit tests for `CometExecRule` [#2863](https://github.com/apache/datafusion-comet/pull/2863) (andygrove) - chore: Add unit tests for `CometScanRule` [#2867](https://github.com/apache/datafusion-comet/pull/2867) (andygrove) - minor: Pedantic refactoring to move some methods from `CometSparkSessionExtensions` to `CometScanRule` and `CometExecRule` [#2873](https://github.com/apache/datafusion-comet/pull/2873) (andygrove) - deps: [iceberg] upgrade DataFusion to 51, Arrow to 57, Iceberg to latest, MSRV to 1.88 [#2729](https://github.com/apache/datafusion-comet/pull/2729) (mbutrovich) - chore: Enable plan stability suite for `native_datafusion` scans [#2877](https://github.com/apache/datafusion-comet/pull/2877) (andygrove) - chore: `ScanExec::new` no longer fetches data [#2881](https://github.com/apache/datafusion-comet/pull/2881) (andygrove) - Chore: refactor bit_not [#2896](https://github.com/apache/datafusion-comet/pull/2896) (kazantsev-maksim) - chore(deps): bump actions/cache from 4 to 5 [#2909](https://github.com/apache/datafusion-comet/pull/2909) (dependabot[bot]) - chore(deps): bump actions/upload-artifact from 5 to 6 [#2910](https://github.com/apache/datafusion-comet/pull/2910) (dependabot[bot]) - chore: Refactor string benchmarks (~10x reduction in LOC) [#2907](https://github.com/apache/datafusion-comet/pull/2907) (andygrove) - chore(deps): bump actions/download-artifact from 6 to 7 [#2908](https://github.com/apache/datafusion-comet/pull/2908) (dependabot[bot]) - chore: use datafusion impl of hex function [#2915](https://github.com/apache/datafusion-comet/pull/2915) (kazantsev-maksim) - chore: Use fixed seed in RNG in tests [#2917](https://github.com/apache/datafusion-comet/pull/2917) (andygrove) - chore: Remove `row_step` from `process_sorted_row_partition` [#2920](https://github.com/apache/datafusion-comet/pull/2920) (andygrove) - chore: Move string function handling to new expression registry [#2931](https://github.com/apache/datafusion-comet/pull/2931) (andygrove) - chore: Reduce syscalls in metrics update logic [#2940](https://github.com/apache/datafusion-comet/pull/2940) (andygrove) - chore: Add shuffle benchmark for deeply nested schemas [#2902](https://github.com/apache/datafusion-comet/pull/2902) (andygrove) - chore: Reduce timer overhead in native shuffle writer [#2941](https://github.com/apache/datafusion-comet/pull/2941) (andygrove) - chore: Remove low-level ffi/jvm timers from native `ScanExec` [#2939](https://github.com/apache/datafusion-comet/pull/2939) (andygrove) - build: Skip problematic Spark SQL test for Spark 4.0.x [#2947](https://github.com/apache/datafusion-comet/pull/2947) (andygrove) - build: Reinstate macOS CI builds of Comet with Spark 4.0 [#2950](https://github.com/apache/datafusion-comet/pull/2950) (manuzhang) - chore(deps): bump reqwest from 0.12.25 to 0.12.26 in /native [#2952](https://github.com/apache/datafusion-comet/pull/2952) (dependabot[bot]) - chore(deps): bump cc from 1.2.49 to 1.2.50 in /native [#2954](https://github.com/apache/datafusion-comet/pull/2954) (dependabot[bot]) - chore(deps): bump assertables from 9.8.2 to 9.8.3 in /native [#2953](https://github.com/apache/datafusion-comet/pull/2953) (dependabot[bot]) - minor: Refactor expression microbenchmarks to remove duplicate code [#2956](https://github.com/apache/datafusion-comet/pull/2956) (andygrove) - build: fix missing import in `main` [#2962](https://github.com/apache/datafusion-comet/pull/2962) (andygrove) - build: Skip macOS Spark 4 fuzz test [#2966](https://github.com/apache/datafusion-comet/pull/2966) (andygrove) - Avoid duplicated writer nodes when AQE enabled [#2982](https://github.com/apache/datafusion-comet/pull/2982) (comphead) - build: Set thread thresholds envs for spark test on macOS [#2987](https://github.com/apache/datafusion-comet/pull/2987) (wForget) - chore: Add microbenchmark for casting string to temporal types [#2980](https://github.com/apache/datafusion-comet/pull/2980) (andygrove) - chore(deps): bump reqwest from 0.12.26 to 0.12.28 in /native [#3009](https://github.com/apache/datafusion-comet/pull/3009) (dependabot[bot]) - chore(deps): bump tempfile from 3.23.0 to 3.24.0 in /native [#3006](https://github.com/apache/datafusion-comet/pull/3006) (dependabot[bot]) - chore(deps): bump serde_json from 1.0.145 to 1.0.148 in /native [#3010](https://github.com/apache/datafusion-comet/pull/3010) (dependabot[bot]) - chore: Add microbenchmark for casting string to numeric [#2979](https://github.com/apache/datafusion-comet/pull/2979) (andygrove) - chore: Skip some CI workflows for benchmark changes [#3030](https://github.com/apache/datafusion-comet/pull/3030) (andygrove) - chore: Skip more workflows on benchmark PRs [#3034](https://github.com/apache/datafusion-comet/pull/3034) (andygrove) - chore: Improve microbenchmark for string expressions [#2964](https://github.com/apache/datafusion-comet/pull/2964) (andygrove) - chore(deps): bump tokio from 1.48.0 to 1.49.0 in /native [#3039](https://github.com/apache/datafusion-comet/pull/3039) (dependabot[bot]) - chore(deps): bump libc from 0.2.178 to 0.2.179 in /native [#3038](https://github.com/apache/datafusion-comet/pull/3038) (dependabot[bot]) - chore(deps): bump actions/cache from 4 to 5 [#3037](https://github.com/apache/datafusion-comet/pull/3037) (dependabot[bot]) - Chore: to_json unit/benchmark tests [#3011](https://github.com/apache/datafusion-comet/pull/3011) (kazantsev-maksim) - chore: Add checks to microbenchmarks for plan running natively in Comet [#3045](https://github.com/apache/datafusion-comet/pull/3045) (andygrove) - chore: Refactor `CometScanRule` to improve scan selection and fallback logic [#2978](https://github.com/apache/datafusion-comet/pull/2978) (andygrove) - chore: Respect to legacySizeOfNull option for size function [#3036](https://github.com/apache/datafusion-comet/pull/3036) (kazantsev-maksim) - chore: Add PySpark-based benchmarks, starting with ETL example [#3065](https://github.com/apache/datafusion-comet/pull/3065) (andygrove) - chore(deps): bump the proto group in /native with 2 updates [#3071](https://github.com/apache/datafusion-comet/pull/3071) (dependabot[bot]) - chore: add MacOS file and event trace log to gitignore [#3070](https://github.com/apache/datafusion-comet/pull/3070) (manuzhang) - chore(deps): bump arrow from 57.1.0 to 57.2.0 in /native [#3073](https://github.com/apache/datafusion-comet/pull/3073) (dependabot[bot]) - chore(deps): bump parquet from 57.1.0 to 57.2.0 in /native [#3074](https://github.com/apache/datafusion-comet/pull/3074) (dependabot[bot]) - chore(deps): bump cc from 1.2.50 to 1.2.52 in /native [#3072](https://github.com/apache/datafusion-comet/pull/3072) (dependabot[bot]) - chore: improve cast documentation to add support per eval mode [#3056](https://github.com/apache/datafusion-comet/pull/3056) (coderfender) - chore: Refactor JVM shuffle: Move `SpillSorter` to top level class and add tests [#3081](https://github.com/apache/datafusion-comet/pull/3081) (andygrove) - minor: Split CometShuffleExternalSorter into sync/async implementations [#3192](https://github.com/apache/datafusion-comet/pull/3192) (andygrove) - chore: Add pending PR shield [#3205](https://github.com/apache/datafusion-comet/pull/3205) (comphead) - chore: deprecate native_comet scan in favor of native_iceberg_compat [#2949](https://github.com/apache/datafusion-comet/pull/2949) (Shekharrajak) - chore: add script to regenerate golden files for plan stability tests [#3204](https://github.com/apache/datafusion-comet/pull/3204) (andygrove) - chore: fix clippy warnings for Rust 1.93 [#3239](https://github.com/apache/datafusion-comet/pull/3239) (andygrove) - build: build native library once and share across CI test jobs [#3249](https://github.com/apache/datafusion-comet/pull/3249) (andygrove) - Experimental: Native CSV files read [#3044](https://github.com/apache/datafusion-comet/pull/3044) (kazantsev-maksim) - build: add missing datafusion-datasource dependency [#3252](https://github.com/apache/datafusion-comet/pull/3252) (andygrove) - chore: Auto scan mode no longer falls back to `native_comet` [#3236](https://github.com/apache/datafusion-comet/pull/3236) (andygrove) - build: optimize CI cache usage and add fast lint gate [#3251](https://github.com/apache/datafusion-comet/pull/3251) (andygrove) - build: use `install` instead of `compile` in TPC CI jobs [#3263](https://github.com/apache/datafusion-comet/pull/3263) (andygrove) - build: remove dead code for 0.8/0.9 docs that broke CI [#3264](https://github.com/apache/datafusion-comet/pull/3264) (andygrove) - refactor: rename scan.allowIncompatible to scan.unsignedSmallIntSafetyCheck [#3238](https://github.com/apache/datafusion-comet/pull/3238) (andygrove) ## Credits Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. ``` 91 Andy Grove 23 dependabot[bot] 18 Matt Butrovich 9 B Vadlamani 7 Oleks V 5 Kazantsev Maksim 5 Manu Zhang 3 Shekhar Prasad Rajak 2 hsiang-c 1 Bjørn Jørgensen 1 Emily Matheys 1 Parth Chandra 1 Raz Luvaton 1 Wonseok Yang 1 Zhen Wang ``` Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. ================================================ FILE: dev/changelog/0.14.0.md ================================================ # DataFusion Comet 0.14.0 Changelog This release consists of 189 commits from 21 contributors. See credits at the end of this changelog for more information. **Fixed bugs:** - fix: [iceberg] Fall back on dynamicpruning expressions for CometIcebergNativeScan [#3335](https://github.com/apache/datafusion-comet/pull/3335) (mbutrovich) - fix: [iceberg] Disable native c2r by default [#3348](https://github.com/apache/datafusion-comet/pull/3348) (andygrove) - fix: Fix `space()` with negative input [#3347](https://github.com/apache/datafusion-comet/pull/3347) (hsiang-c) - fix: respect scan impl config for v2 scan [#3357](https://github.com/apache/datafusion-comet/pull/3357) (andygrove) - fix: fix memory safety issue in native c2r [#3367](https://github.com/apache/datafusion-comet/pull/3367) (andygrove) - fix: preserve partitioning in CometNativeScanExec for bucketed scans [#3392](https://github.com/apache/datafusion-comet/pull/3392) (andygrove) - fix: unignore row index Spark SQL tests for native_datafusion [#3414](https://github.com/apache/datafusion-comet/pull/3414) (andygrove) - fix: fall back to Spark when Parquet field ID matching is enabled in native_datafusion [#3415](https://github.com/apache/datafusion-comet/pull/3415) (andygrove) - fix: Expose bucketing information from CometNativeScanExec [#3437](https://github.com/apache/datafusion-comet/pull/3437) (andygrove) - fix: support scalar processing for `space` function [#3408](https://github.com/apache/datafusion-comet/pull/3408) (kazantsev-maksim) - fix: Revert "perf: Remove mutable buffers from scan partition/missing columns (#3411)" [iceberg] [#3486](https://github.com/apache/datafusion-comet/pull/3486) (mbutrovich) - fix: unignore input_file_name Spark SQL tests for native_datafusion [#3458](https://github.com/apache/datafusion-comet/pull/3458) (andygrove) - fix: add scalar support for bit_count expression [#3361](https://github.com/apache/datafusion-comet/pull/3361) (hsiang-c) - fix: Support concat_ws with literal NULL separator [#3542](https://github.com/apache/datafusion-comet/pull/3542) (0lai0) - fix: handle type mismatches in native c2r conversion [#3583](https://github.com/apache/datafusion-comet/pull/3583) (andygrove) - fix: disable native C2R for legacy Iceberg scans [iceberg] [#3663](https://github.com/apache/datafusion-comet/pull/3663) (mbutrovich) - fix: resolve Miri UB in null struct field test, re-enable Miri on PRs [#3669](https://github.com/apache/datafusion-comet/pull/3669) (andygrove) - fix: Support on all-literal RLIKE expression [#3647](https://github.com/apache/datafusion-comet/pull/3647) (0lai0) - fix: Fix scan metrics test to run with both native_datafusion and native_iceberg_compat [#3690](https://github.com/apache/datafusion-comet/pull/3690) (andygrove) **Performance related:** - perf: refactor sum int with specialized implementations for each eval_mode [#3054](https://github.com/apache/datafusion-comet/pull/3054) (andygrove) - perf: Optimize contains expression with SIMD-based scalar pattern sea… [#2991](https://github.com/apache/datafusion-comet/pull/2991) (Shekharrajak) - perf: Add batch coalescing in BufBatchWriter to reduce IPC schema overhead [#3441](https://github.com/apache/datafusion-comet/pull/3441) (andygrove) - perf: Use `native_datafusion` scan in benchmark scripts (6% faster for TPC-H) [#3460](https://github.com/apache/datafusion-comet/pull/3460) (andygrove) - perf: Remove mutable buffers from scan partition/missing columns [#3411](https://github.com/apache/datafusion-comet/pull/3411) (andygrove) - perf: [iceberg] Single-pass FileScanTask validation [#3443](https://github.com/apache/datafusion-comet/pull/3443) (mbutrovich) - perf: Improve benchmarks for native row-to-columnar used by JVM shuffle [#3290](https://github.com/apache/datafusion-comet/pull/3290) (andygrove) - perf: executePlan uses a channel to park executor task thread instead of yield_now() [iceberg] [#3553](https://github.com/apache/datafusion-comet/pull/3553) (mbutrovich) - perf: Initialize tokio runtime worker threads from spark.executor.cores [#3555](https://github.com/apache/datafusion-comet/pull/3555) (andygrove) - perf: Add Comet config for native Iceberg reader's data file concurrency [iceberg] [#3584](https://github.com/apache/datafusion-comet/pull/3584) (mbutrovich) - perf: reuse CometConf.COMET_TRACING_ENABLED, Native, NativeUtil in NativeBatchDecoderIterator [#3627](https://github.com/apache/datafusion-comet/pull/3627) (mbutrovich) - perf: Improve performance of native row-to-columnar transition used by JVM shuffle [#3289](https://github.com/apache/datafusion-comet/pull/3289) (andygrove) - perf: use aligned pointer reads for SparkUnsafeRow field accessors [#3670](https://github.com/apache/datafusion-comet/pull/3670) (andygrove) - perf: Optimize some decimal expressions [#3619](https://github.com/apache/datafusion-comet/pull/3619) (andygrove) **Implemented enhancements:** - feat: Native columnar to row conversion (Phase 2) [#3266](https://github.com/apache/datafusion-comet/pull/3266) (andygrove) - feat: Enable native columnar-to-row by default [#3299](https://github.com/apache/datafusion-comet/pull/3299) (andygrove) - feat: add support for `width_bucket` expression [#3273](https://github.com/apache/datafusion-comet/pull/3273) (davidlghellin) - feat: Drop `native_comet` as a valid option for `COMET_NATIVE_SCAN_IMPL` config [#3358](https://github.com/apache/datafusion-comet/pull/3358) (andygrove) - feat: Support date to timestamp cast [#3383](https://github.com/apache/datafusion-comet/pull/3383) (coderfender) - feat: CometExecRDD supports per-partition plan data, reduce Iceberg native scan serialization, add DPP [iceberg] [#3349](https://github.com/apache/datafusion-comet/pull/3349) (mbutrovich) - feat: Support right expression [#3207](https://github.com/apache/datafusion-comet/pull/3207) (Shekharrajak) - feat: support map_contains_key expression [#3369](https://github.com/apache/datafusion-comet/pull/3369) (peterxcli) - feat: add support for make_date expression [#3147](https://github.com/apache/datafusion-comet/pull/3147) (andygrove) - feat: add support for next_day expression [#3148](https://github.com/apache/datafusion-comet/pull/3148) (andygrove) - feat: implement cast from whole numbers to binary format and bool to decimal [#3083](https://github.com/apache/datafusion-comet/pull/3083) (coderfender) - feat: Support for StringSplit [#2772](https://github.com/apache/datafusion-comet/pull/2772) (Shekharrajak) - feat: CometNativeScan per-partition plan serde [#3511](https://github.com/apache/datafusion-comet/pull/3511) (mbutrovich) - feat: Remove mutable buffers from scan partition/missing columns [iceberg] [#3514](https://github.com/apache/datafusion-comet/pull/3514) (andygrove) - feat: pass spark.comet.datafusion.\* configs through to DataFusion session [#3455](https://github.com/apache/datafusion-comet/pull/3455) (andygrove) - feat: pass vended credentials to Iceberg native scan [#3523](https://github.com/apache/datafusion-comet/pull/3523) (tokoko) - feat: Cast date to Numeric (No Op) [#3544](https://github.com/apache/datafusion-comet/pull/3544) (coderfender) - feat: add support `crc32` expression [#3498](https://github.com/apache/datafusion-comet/pull/3498) (rafafrdz) - feat: Support int to timestamp casts [#3541](https://github.com/apache/datafusion-comet/pull/3541) (coderfender) - feat(benchmarks): add async-profiler support to TPC benchmark scripts [#3613](https://github.com/apache/datafusion-comet/pull/3613) (andygrove) - feat: Cast numeric (non int) to timestamp [#3559](https://github.com/apache/datafusion-comet/pull/3559) (coderfender) - feat: [ANSI] Ansi sql error messages [#3580](https://github.com/apache/datafusion-comet/pull/3580) (parthchandra) - feat: enable debug assertions in CI profile, fix unaligned memory access bug [#3652](https://github.com/apache/datafusion-comet/pull/3652) (andygrove) - feat: Enable native c2r by default, add debug asserts [#3649](https://github.com/apache/datafusion-comet/pull/3649) (andygrove) - feat: support Spark luhn_check expression [#3573](https://github.com/apache/datafusion-comet/pull/3573) (n0r0shi) **Documentation updates:** - docs: Add changelog for 0.13.0 [#3260](https://github.com/apache/datafusion-comet/pull/3260) (andygrove) - docs: fix bug in placement of prettier-ignore-end in generated docs [#3287](https://github.com/apache/datafusion-comet/pull/3287) (andygrove) - docs: Add contributor guide page for SQL file tests [#3333](https://github.com/apache/datafusion-comet/pull/3333) (andygrove) - docs: fix inaccurate claim about mutable buffers in parquet scan docs [#3378](https://github.com/apache/datafusion-comet/pull/3378) (andygrove) - docs: Improve documentation on maven usage for running tests [#3370](https://github.com/apache/datafusion-comet/pull/3370) (andygrove) - docs: move release process docs to contributor guide [#3492](https://github.com/apache/datafusion-comet/pull/3492) (andygrove) - docs: improve release process documentation [#3508](https://github.com/apache/datafusion-comet/pull/3508) (andygrove) - docs: update roadmap [#3543](https://github.com/apache/datafusion-comet/pull/3543) (mbutrovich) - docs: Update Parquet scan documentation [#3433](https://github.com/apache/datafusion-comet/pull/3433) (andygrove) - docs: recommend SQL file tests for new expressions [#3598](https://github.com/apache/datafusion-comet/pull/3598) (andygrove) - docs: add SAFETY comments to all unsafe blocks in shuffle spark_unsafe module [#3603](https://github.com/apache/datafusion-comet/pull/3603) (andygrove) - docs: Fix link to overview page [#3625](https://github.com/apache/datafusion-comet/pull/3625) (manuzhang) - doc: Document sql query error propagation [#3651](https://github.com/apache/datafusion-comet/pull/3651) (parthchandra) - docs: update Iceberg docs in advance of 0.14.0 [#3691](https://github.com/apache/datafusion-comet/pull/3691) (mbutrovich) **Other:** - chore(deps): bump actions/download-artifact from 4 to 7 [#3281](https://github.com/apache/datafusion-comet/pull/3281) (dependabot[bot]) - chore(deps): bump cc from 1.2.53 to 1.2.54 in /native [#3284](https://github.com/apache/datafusion-comet/pull/3284) (dependabot[bot]) - build: Fix docs workflow dependency resolution failure [#3275](https://github.com/apache/datafusion-comet/pull/3275) (andygrove) - chore(deps): bump actions/upload-artifact from 4 to 6 [#3280](https://github.com/apache/datafusion-comet/pull/3280) (dependabot[bot]) - chore(deps): bump actions/cache from 4 to 5 [#3279](https://github.com/apache/datafusion-comet/pull/3279) (dependabot[bot]) - chore(deps): bump uuid from 1.19.0 to 1.20.0 in /native [#3282](https://github.com/apache/datafusion-comet/pull/3282) (dependabot[bot]) - build: reduce overhead of fuzz testing [#3257](https://github.com/apache/datafusion-comet/pull/3257) (andygrove) - chore: Start 0.14.0 development [#3288](https://github.com/apache/datafusion-comet/pull/3288) (andygrove) - chore: Add Comet released artifacts and links to maven [#3291](https://github.com/apache/datafusion-comet/pull/3291) (comphead) - chore: Add take/untake workflow for issue self-assignment [#3270](https://github.com/apache/datafusion-comet/pull/3270) (andygrove) - ci: Consolidate Spark SQL test jobs to reduce CI time [#3271](https://github.com/apache/datafusion-comet/pull/3271) (andygrove) - chore(deps): bump org.assertj:assertj-core from 3.23.1 to 3.27.7 [#3293](https://github.com/apache/datafusion-comet/pull/3293) (dependabot[bot]) - chore: Add microbenchmark for IcebergScan operator serde roundtrip [#3296](https://github.com/apache/datafusion-comet/pull/3296) (andygrove) - chore: Remove IgnoreCometNativeScan from ParquetEncryptionSuite in 3.5.7 diff [#3304](https://github.com/apache/datafusion-comet/pull/3304) (andygrove) - chore: Enable native c2r in plan stability suite [#3302](https://github.com/apache/datafusion-comet/pull/3302) (andygrove) - chore: Add support for Spark 3.5.8 [#3323](https://github.com/apache/datafusion-comet/pull/3323) (manuzhang) - chore: Invert usingDataSourceExec test helper to usingLegacyNativeCometScan [#3310](https://github.com/apache/datafusion-comet/pull/3310) (andygrove) - tests: Add SQL test files covering edge cases for (almost) every Comet-supported expression [#3328](https://github.com/apache/datafusion-comet/pull/3328) (andygrove) - chore: Adapt caching from #3251 to [iceberg] workflows [#3353](https://github.com/apache/datafusion-comet/pull/3353) (mbutrovich) - bug: Fix string decimal type throw right exception [#3248](https://github.com/apache/datafusion-comet/pull/3248) (coderfender) - chore: Migrate `concat` tests to sql based testing framework [#3352](https://github.com/apache/datafusion-comet/pull/3352) (andygrove) - chore(deps): bump actions/setup-java from 4 to 5 [#3363](https://github.com/apache/datafusion-comet/pull/3363) (dependabot[bot]) - chore: Annotate classes/methods/fields that are used by Apache Iceberg [#3237](https://github.com/apache/datafusion-comet/pull/3237) (andygrove) - Feat: map_from_entries [#2905](https://github.com/apache/datafusion-comet/pull/2905) (kazantsev-maksim) - chore: Move spark unsafe classes into spark_unsafe [#3373](https://github.com/apache/datafusion-comet/pull/3373) (EmilyMatt) - chore: Extract some tied down logic [#3374](https://github.com/apache/datafusion-comet/pull/3374) (EmilyMatt) - Fix: array contains null handling [#3372](https://github.com/apache/datafusion-comet/pull/3372) (Shekharrajak) - chore: stop uploading code coverage results [#3381](https://github.com/apache/datafusion-comet/pull/3381) (andygrove) - chore: update target-cpus in published binaries to x86-64-v3 and neoverse-n1 [#3368](https://github.com/apache/datafusion-comet/pull/3368) (mbutrovich) - chore: show line of error sql [#3390](https://github.com/apache/datafusion-comet/pull/3390) (peterxcli) - chore: Move writer-related logic to "writers" module [#3385](https://github.com/apache/datafusion-comet/pull/3385) (EmilyMatt) - chore(deps): bump bytes from 1.11.0 to 1.11.1 in /native [#3380](https://github.com/apache/datafusion-comet/pull/3380) (dependabot[bot]) - chore: Clean up and split shuffle module [#3395](https://github.com/apache/datafusion-comet/pull/3395) (EmilyMatt) - chore: Make PR workflows match target-cpu flags in published jars [#3402](https://github.com/apache/datafusion-comet/pull/3402) (mbutrovich) - chore(deps): bump time from 0.3.45 to 0.3.47 in /native [#3412](https://github.com/apache/datafusion-comet/pull/3412) (dependabot[bot]) - chore: Run Spark SQL tests with `native_datafusion` in CI [#3393](https://github.com/apache/datafusion-comet/pull/3393) (andygrove) - test: Add ANSI mode SQL test files for expressions that throw on invalid input [#3377](https://github.com/apache/datafusion-comet/pull/3377) (andygrove) - refactor: Split read benchmarks and add addParquetScanCases helper [#3407](https://github.com/apache/datafusion-comet/pull/3407) (andygrove) - chore: 4.5x reduction in number of golden files [#3399](https://github.com/apache/datafusion-comet/pull/3399) (andygrove) - Feat: to_csv [#3004](https://github.com/apache/datafusion-comet/pull/3004) (kazantsev-maksim) - minor: map_from_entries sql tests [#3394](https://github.com/apache/datafusion-comet/pull/3394) (kazantsev-maksim) - chore: add confirmation before tarball is released [#3439](https://github.com/apache/datafusion-comet/pull/3439) (milenkovicm) - chore(deps): bump cc from 1.2.54 to 1.2.55 in /native [#3451](https://github.com/apache/datafusion-comet/pull/3451) (dependabot[bot]) - chore: Add Iceberg TPC-H benchmarking scripts [#3294](https://github.com/apache/datafusion-comet/pull/3294) (andygrove) - chore: Remove dead code paths for deprecated native_comet scan [#3396](https://github.com/apache/datafusion-comet/pull/3396) (andygrove) - chore(deps): bump arrow from 57.2.0 to 57.3.0 in /native [#3449](https://github.com/apache/datafusion-comet/pull/3449) (dependabot[bot]) - chore(deps): bump aws-config from 1.8.12 to 1.8.13 in /native [#3450](https://github.com/apache/datafusion-comet/pull/3450) (dependabot[bot]) - chore(deps): bump regex from 1.12.2 to 1.12.3 in /native [#3453](https://github.com/apache/datafusion-comet/pull/3453) (dependabot[bot]) - chore(deps): bump rand from 0.9.2 to 0.10.0 in /native [#3465](https://github.com/apache/datafusion-comet/pull/3465) (manuzhang) - test: Add additional contains expression tests [#3462](https://github.com/apache/datafusion-comet/pull/3462) (andygrove) - chore: Adjust native artifact caching key in CI [#3476](https://github.com/apache/datafusion-comet/pull/3476) (mbutrovich) - chore: Add Comet writer nested types test assertion [#3480](https://github.com/apache/datafusion-comet/pull/3480) (comphead) - test: Add SQL file tests for left and right expressions [#3463](https://github.com/apache/datafusion-comet/pull/3463) (andygrove) - chore: Add GitHub workflow to close stale PRs [#3488](https://github.com/apache/datafusion-comet/pull/3488) (andygrove) - chore: Make `push` CI to be triggered for `main` branch only [#3474](https://github.com/apache/datafusion-comet/pull/3474) (comphead) - ci: disable Miri safety checks until compatibility is restored [#3504](https://github.com/apache/datafusion-comet/pull/3504) (andygrove) - chore: Add memory reservation debug logging [#3489](https://github.com/apache/datafusion-comet/pull/3489) (andygrove) - chore: enable GitHub button for updating PR branches with latest from main [#3505](https://github.com/apache/datafusion-comet/pull/3505) (andygrove) - chore: remove some dead cast code [#3513](https://github.com/apache/datafusion-comet/pull/3513) (andygrove) - chore(deps): bump aws-credential-types from 1.2.11 to 1.2.12 in /native [#3525](https://github.com/apache/datafusion-comet/pull/3525) (dependabot[bot]) - chore(deps): bump libc from 0.2.180 to 0.2.182 in /native [#3527](https://github.com/apache/datafusion-comet/pull/3527) (dependabot[bot]) - chore(deps): bump cc from 1.2.55 to 1.2.56 in /native [#3528](https://github.com/apache/datafusion-comet/pull/3528) (dependabot[bot]) - chore(deps): bump tempfile from 3.24.0 to 3.25.0 in /native [#3529](https://github.com/apache/datafusion-comet/pull/3529) (dependabot[bot]) - ci: Bump up `actions/upload-artifact` from v4 to v6 [#3533](https://github.com/apache/datafusion-comet/pull/3533) (manuzhang) - chore(deps): bump aws-config from 1.8.13 to 1.8.14 in /native [#3526](https://github.com/apache/datafusion-comet/pull/3526) (dependabot[bot]) - chore: refactor array_repeat [#3516](https://github.com/apache/datafusion-comet/pull/3516) (kazantsev-maksim) - chore: Add envvars to override writer configs and cometConf minor clean up [#3540](https://github.com/apache/datafusion-comet/pull/3540) (comphead) - chore: Cast module refactor boolean module [#3491](https://github.com/apache/datafusion-comet/pull/3491) (coderfender) - chore: Consolidate TPC benchmark scripts [#3538](https://github.com/apache/datafusion-comet/pull/3538) (andygrove) - chore(deps): bump parquet from 57.2.0 to 57.3.0 in /native [#3568](https://github.com/apache/datafusion-comet/pull/3568) (dependabot[bot]) - chore(deps): bump uuid from 1.20.0 to 1.21.0 in /native [#3567](https://github.com/apache/datafusion-comet/pull/3567) (dependabot[bot]) - chore: Add TPC-\* queries to repo [#3562](https://github.com/apache/datafusion-comet/pull/3562) (andygrove) - chore(deps): bump assertables from 9.8.4 to 9.8.6 in /native [#3570](https://github.com/apache/datafusion-comet/pull/3570) (dependabot[bot]) - chore(deps): bump actions/stale from 10.1.1 to 10.2.0 [#3565](https://github.com/apache/datafusion-comet/pull/3565) (dependabot[bot]) - chore(deps): bump aws-credential-types from 1.2.12 to 1.2.13 in /native [#3566](https://github.com/apache/datafusion-comet/pull/3566) (dependabot[bot]) - chore: makes dependabot to group deps into single PR [#3578](https://github.com/apache/datafusion-comet/pull/3578) (comphead) - chore: Cast module refactor : String [#3577](https://github.com/apache/datafusion-comet/pull/3577) (coderfender) - chore(deps): bump the all-other-cargo-deps group in /native with 3 updates [#3581](https://github.com/apache/datafusion-comet/pull/3581) (dependabot[bot]) - chore: Add Docker Compose support for TPC benchmarks [#3576](https://github.com/apache/datafusion-comet/pull/3576) (andygrove) - build: Runs-on for `PR Build (Linux)` [#3579](https://github.com/apache/datafusion-comet/pull/3579) (blaginin) - chore: Add consistency checks and result hashing to TPC benchmarks [#3582](https://github.com/apache/datafusion-comet/pull/3582) (andygrove) - chore: Remove all remaining uses of legacy BatchReader from Comet [iceberg] [#3468](https://github.com/apache/datafusion-comet/pull/3468) (andygrove) - build: Skip CI workflows for changes in benchmarks directory [#3599](https://github.com/apache/datafusion-comet/pull/3599) (andygrove) - build: fix runs-on tags for consistency [#3601](https://github.com/apache/datafusion-comet/pull/3601) (andygrove) - chore: Add Java Flight Recorder profiling to TPC benchmarks [#3597](https://github.com/apache/datafusion-comet/pull/3597) (andygrove) - deps: DataFusion 52.0.0 migration (SchemaAdapter changes, etc.) [iceberg] [#3536](https://github.com/apache/datafusion-comet/pull/3536) (comphead) - chore(deps): bump actions/download-artifact from 7 to 8 [#3609](https://github.com/apache/datafusion-comet/pull/3609) (dependabot[bot]) - chore(deps): bump actions/upload-artifact from 6 to 7 [#3610](https://github.com/apache/datafusion-comet/pull/3610) (dependabot[bot]) - chore: bump iceberg-rust dependency to latest [iceberg] [#3606](https://github.com/apache/datafusion-comet/pull/3606) (mbutrovich) - CI: Add CodeQL workflow for GitHub Actions security scanning [#3617](https://github.com/apache/datafusion-comet/pull/3617) (kevinjqliu) - CI: update codeql with pinned action versions [#3621](https://github.com/apache/datafusion-comet/pull/3621) (kevinjqliu) - chore: replace legacy datetime rebase tests with current scan coverage [iceberg] [#3605](https://github.com/apache/datafusion-comet/pull/3605) (andygrove) - build: More runners [#3626](https://github.com/apache/datafusion-comet/pull/3626) (blaginin) - deps: bump DataFusion to 52.2 [iceberg] [#3622](https://github.com/apache/datafusion-comet/pull/3622) (mbutrovich) - chore: use datafusion impl of `space` function [#3612](https://github.com/apache/datafusion-comet/pull/3612) (kazantsev-maksim) - chore: use datafusion impl of `bit_count` function [#3616](https://github.com/apache/datafusion-comet/pull/3616) (kazantsev-maksim) - chore: refactor cast module numeric data types [#3623](https://github.com/apache/datafusion-comet/pull/3623) (coderfender) - chore: Refactor cast module temporal types [#3624](https://github.com/apache/datafusion-comet/pull/3624) (coderfender) - chore: Fix clippy complaints [#3634](https://github.com/apache/datafusion-comet/pull/3634) (comphead) - chore(deps): bump docker/build-push-action from 6 to 7 [#3639](https://github.com/apache/datafusion-comet/pull/3639) (dependabot[bot]) - chore(deps): bump github/codeql-action from 4.32.5 to 4.32.6 [#3637](https://github.com/apache/datafusion-comet/pull/3637) (dependabot[bot]) - chore(deps): bump docker/setup-buildx-action from 3 to 4 [#3636](https://github.com/apache/datafusion-comet/pull/3636) (dependabot[bot]) - chore(deps): bump docker/login-action from 3 to 4 [#3638](https://github.com/apache/datafusion-comet/pull/3638) (dependabot[bot]) - deps: update to latest iceberg-rust to pick up get_byte_ranges [iceberg] [#3635](https://github.com/apache/datafusion-comet/pull/3635) (mbutrovich) - chore: Array literals tests enable [#3633](https://github.com/apache/datafusion-comet/pull/3633) (comphead) - chore: Add debug assertions before unsafe code blocks [#3655](https://github.com/apache/datafusion-comet/pull/3655) (andygrove) - chore: fix license header - ansi docs [#3662](https://github.com/apache/datafusion-comet/pull/3662) (coderfender) - chore(deps): bump quinn-proto from 0.11.13 to 0.11.14 in /native [#3660](https://github.com/apache/datafusion-comet/pull/3660) (dependabot[bot]) - ci: add dedicated RAT license check workflow for all PRs [#3664](https://github.com/apache/datafusion-comet/pull/3664) (andygrove) - chore: Remove deprecated SCAN_NATIVE_COMET constant and related test code [#3671](https://github.com/apache/datafusion-comet/pull/3671) (andygrove) - chore: Upgrade to DF 52.3.0 [#3672](https://github.com/apache/datafusion-comet/pull/3672) (andygrove) - deps: update to iceberg-rust 0.9.0 rc1 [iceberg] [#3657](https://github.com/apache/datafusion-comet/pull/3657) (mbutrovich) - chore: Mark expressions with known correctness issues as incompatible [#3675](https://github.com/apache/datafusion-comet/pull/3675) (andygrove) - chore(deps): bump actions/setup-java from 4 to 5 [#3683](https://github.com/apache/datafusion-comet/pull/3683) (dependabot[bot]) - chore(deps): bump runs-on/action from 2.0.3 to 2.1.0 [#3684](https://github.com/apache/datafusion-comet/pull/3684) (dependabot[bot]) - chore(deps): bump actions/checkout from 4 to 6 [#3685](https://github.com/apache/datafusion-comet/pull/3685) (dependabot[bot]) - ci: remove Java Iceberg integration tests from CI [iceberg] [#3673](https://github.com/apache/datafusion-comet/pull/3673) (andygrove) ## Credits Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. ``` 81 Andy Grove 34 dependabot[bot] 19 Matt Butrovich 9 B Vadlamani 8 Oleks V 7 Kazantsev Maksim 4 Emily Matheys 4 Manu Zhang 4 Shekhar Prasad Rajak 2 Bhargava Vadlamani 2 ChenChen Lai 2 Dmitrii Blaginin 2 Kevin Liu 2 Parth Chandra 2 Peter Lee 2 hsiang-c 1 David López 1 Marko Milenković 1 Rafael Fernández 1 Tornike Gurgenidze 1 n0r0shi ``` Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. ================================================ FILE: dev/changelog/0.14.1.md ================================================ # DataFusion Comet 0.14.1 Changelog This release consists of 5 commits from 1 contributors. See credits at the end of this changelog for more information. **Fixed bugs:** - fix: [branch-0.14] backport #3802 - cache object stores and bucket regions to reduce DNS query volume [#3935](https://github.com/apache/datafusion-comet/pull/3935) (andygrove) - fix: [branch-0.14] backport #3924 - share unified memory pools across native execution contexts [#3938](https://github.com/apache/datafusion-comet/pull/3938) (andygrove) - fix: [branch-0.14] backport #3879 - skip Comet columnar shuffle for stages with DPP scans [#3934](https://github.com/apache/datafusion-comet/pull/3934) (andygrove) - fix: [branch-0.14] backport #3914 - use min instead of max when capping write buffer size to Int range [#3936](https://github.com/apache/datafusion-comet/pull/3936) (andygrove) - fix: [branch-0.14] backport #3865 - handle ambiguous and non-existent local times [#3937](https://github.com/apache/datafusion-comet/pull/3937) (andygrove) ## Credits Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. ``` 5 Andy Grove ``` Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. ================================================ FILE: dev/changelog/0.2.0.md ================================================ # DataFusion Comet 0.2.0 Changelog This release consists of 87 commits from 14 contributors. See credits at the end of this changelog for more information. **Fixed bugs:** - fix: dictionary decimal vector optimization [#705](https://github.com/apache/datafusion-comet/pull/705) (kazuyukitanimura) - fix: Unsupported window expression should fall back to Spark [#710](https://github.com/apache/datafusion-comet/pull/710) (viirya) - fix: ReusedExchangeExec can be child operator of CometBroadcastExchangeExec [#713](https://github.com/apache/datafusion-comet/pull/713) (viirya) - fix: Fallback to Spark for window expression with range frame [#719](https://github.com/apache/datafusion-comet/pull/719) (viirya) - fix: Remove `skip.surefire.tests` mvn property [#739](https://github.com/apache/datafusion-comet/pull/739) (wForget) - fix: subquery execution under CometTakeOrderedAndProjectExec should not fail [#748](https://github.com/apache/datafusion-comet/pull/748) (viirya) - fix: skip negative scale checks for creating decimals [#723](https://github.com/apache/datafusion-comet/pull/723) (kazuyukitanimura) - fix: Fallback to Spark for unsupported partitioning [#759](https://github.com/apache/datafusion-comet/pull/759) (viirya) - fix: Unsupported types for SinglePartition should fallback to Spark [#765](https://github.com/apache/datafusion-comet/pull/765) (viirya) - fix: unwrap dictionaries in CreateNamedStruct [#754](https://github.com/apache/datafusion-comet/pull/754) (andygrove) - fix: Fallback to Spark for unsupported input besides ordering [#768](https://github.com/apache/datafusion-comet/pull/768) (viirya) - fix: Native window operator should be CometUnaryExec [#774](https://github.com/apache/datafusion-comet/pull/774) (viirya) - fix: Fallback to Spark when shuffling on struct with duplicate field name [#776](https://github.com/apache/datafusion-comet/pull/776) (viirya) - fix: withInfo was overwriting information in some cases [#780](https://github.com/apache/datafusion-comet/pull/780) (andygrove) - fix: Improve support for nested structs [#800](https://github.com/apache/datafusion-comet/pull/800) (eejbyfeldt) - fix: Sort on single struct should fallback to Spark [#811](https://github.com/apache/datafusion-comet/pull/811) (viirya) - fix: Check sort order of SortExec instead of child output [#821](https://github.com/apache/datafusion-comet/pull/821) (viirya) - fix: Fix panic in `avg` aggregate and disable `stddev` by default [#819](https://github.com/apache/datafusion-comet/pull/819) (andygrove) - fix: Supported nested types in HashJoin [#735](https://github.com/apache/datafusion-comet/pull/735) (eejbyfeldt) **Performance related:** - perf: Improve performance of CASE .. WHEN expressions [#703](https://github.com/apache/datafusion-comet/pull/703) (andygrove) - perf: Optimize IfExpr by delegating to CaseExpr [#681](https://github.com/apache/datafusion-comet/pull/681) (andygrove) - fix: optimize isNullAt [#732](https://github.com/apache/datafusion-comet/pull/732) (kazuyukitanimura) - perf: decimal decode improvements [#727](https://github.com/apache/datafusion-comet/pull/727) (parthchandra) - fix: Remove castting on decimals with a small precision to decimal256 [#741](https://github.com/apache/datafusion-comet/pull/741) (kazuyukitanimura) - fix: optimize some bit functions [#718](https://github.com/apache/datafusion-comet/pull/718) (kazuyukitanimura) - fix: Optimize getDecimal for small precision [#758](https://github.com/apache/datafusion-comet/pull/758) (kazuyukitanimura) - perf: add metrics to CopyExec and ScanExec [#778](https://github.com/apache/datafusion-comet/pull/778) (andygrove) - fix: Optimize decimal creation macros [#764](https://github.com/apache/datafusion-comet/pull/764) (kazuyukitanimura) - perf: Improve count aggregate performance [#784](https://github.com/apache/datafusion-comet/pull/784) (andygrove) - fix: Optimize read_side_padding [#772](https://github.com/apache/datafusion-comet/pull/772) (kazuyukitanimura) - perf: Remove some redundant copying of batches [#816](https://github.com/apache/datafusion-comet/pull/816) (andygrove) - perf: Remove redundant copying of batches after FilterExec [#835](https://github.com/apache/datafusion-comet/pull/835) (andygrove) - fix: Optimize CheckOverflow [#852](https://github.com/apache/datafusion-comet/pull/852) (kazuyukitanimura) - perf: Add benchmarks for Spark Scan + Comet Exec [#863](https://github.com/apache/datafusion-comet/pull/863) (andygrove) **Implemented enhancements:** - feat: Add support for time-zone, 3 & 5 digit years: Cast from string to timestamp. [#704](https://github.com/apache/datafusion-comet/pull/704) (akhilss99) - feat: Support count AggregateUDF for window function [#736](https://github.com/apache/datafusion-comet/pull/736) (huaxingao) - feat: Implement basic version of RLIKE [#734](https://github.com/apache/datafusion-comet/pull/734) (andygrove) - feat: show executed native plan with metrics when in debug mode [#746](https://github.com/apache/datafusion-comet/pull/746) (andygrove) - feat: Add GetStructField expression [#731](https://github.com/apache/datafusion-comet/pull/731) (Kimahriman) - feat: Add config to enable native upper and lower string conversion [#767](https://github.com/apache/datafusion-comet/pull/767) (andygrove) - feat: Improve native explain [#795](https://github.com/apache/datafusion-comet/pull/795) (andygrove) - feat: Add support for null literal with struct type [#797](https://github.com/apache/datafusion-comet/pull/797) (eejbyfeldt) - feat: Optimze CreateNamedStruct preserve dictionaries [#789](https://github.com/apache/datafusion-comet/pull/789) (eejbyfeldt) - feat: `CreateArray` support [#793](https://github.com/apache/datafusion-comet/pull/793) (Kimahriman) - feat: Add native thread configs [#828](https://github.com/apache/datafusion-comet/pull/828) (viirya) - feat: Add specific configs for converting Spark Parquet and JSON data to Arrow [#832](https://github.com/apache/datafusion-comet/pull/832) (andygrove) - feat: Support sum in window function [#802](https://github.com/apache/datafusion-comet/pull/802) (huaxingao) - feat: Simplify configs for enabling/disabling operators [#855](https://github.com/apache/datafusion-comet/pull/855) (andygrove) - feat: Enable `clippy::clone_on_ref_ptr` on `proto` and `spark_exprs` crates [#859](https://github.com/apache/datafusion-comet/pull/859) (comphead) - feat: Enable `clippy::clone_on_ref_ptr` on `core` crate [#860](https://github.com/apache/datafusion-comet/pull/860) (comphead) - feat: Use CometPlugin as main entrypoint [#853](https://github.com/apache/datafusion-comet/pull/853) (andygrove) **Documentation updates:** - doc: Update outdated spark.comet.columnar.shuffle.enabled configuration doc [#738](https://github.com/apache/datafusion-comet/pull/738) (wForget) - docs: Add explicit configs for enabling operators [#801](https://github.com/apache/datafusion-comet/pull/801) (andygrove) - doc: Document CometPlugin to start Comet in cluster mode [#836](https://github.com/apache/datafusion-comet/pull/836) (comphead) **Other:** - chore: Make rust clippy happy [#701](https://github.com/apache/datafusion-comet/pull/701) (Xuanwo) - chore: Update version to 0.2.0 and add 0.1.0 changelog [#696](https://github.com/apache/datafusion-comet/pull/696) (andygrove) - chore: Use rust-toolchain.toml for better toolchain support [#699](https://github.com/apache/datafusion-comet/pull/699) (Xuanwo) - chore(native): Make sure all targets in workspace been covered by clippy [#702](https://github.com/apache/datafusion-comet/pull/702) (Xuanwo) - Apache DataFusion Comet Logo [#697](https://github.com/apache/datafusion-comet/pull/697) (aocsa) - chore: Add logo to rat exclude list [#709](https://github.com/apache/datafusion-comet/pull/709) (andygrove) - chore: Use new logo in README and website [#724](https://github.com/apache/datafusion-comet/pull/724) (andygrove) - build: Add Comet logo files into exclude list [#726](https://github.com/apache/datafusion-comet/pull/726) (viirya) - chore: Remove TPC-DS benchmark results [#728](https://github.com/apache/datafusion-comet/pull/728) (andygrove) - chore: make Cast's logic reusable for other projects [#716](https://github.com/apache/datafusion-comet/pull/716) (Blizzara) - chore: move scalar_funcs into spark-expr [#712](https://github.com/apache/datafusion-comet/pull/712) (Blizzara) - chore: Bump DataFusion to rev 35c2e7e [#740](https://github.com/apache/datafusion-comet/pull/740) (andygrove) - chore: add more aggregate functions to benchmark test [#706](https://github.com/apache/datafusion-comet/pull/706) (huaxingao) - chore: Add criterion benchmark for decimal_div [#743](https://github.com/apache/datafusion-comet/pull/743) (andygrove) - build: Re-enable TPCDS q72 for broadcast and hash join configs [#781](https://github.com/apache/datafusion-comet/pull/781) (viirya) - chore: bump DataFusion to rev f4e519f [#783](https://github.com/apache/datafusion-comet/pull/783) (huaxingao) - chore: Upgrade to DataFusion rev bddb641 and disable "skip partial aggregates" feature [#788](https://github.com/apache/datafusion-comet/pull/788) (andygrove) - chore: Remove legacy code for adding a cast to a coalesce [#790](https://github.com/apache/datafusion-comet/pull/790) (andygrove) - chore: Use DataFusion 41.0.0-rc1 [#794](https://github.com/apache/datafusion-comet/pull/794) (andygrove) - chore: rename `CometRowToColumnar` and fix duplication bug [#785](https://github.com/apache/datafusion-comet/pull/785) (Kimahriman) - chore: Enable shuffle in micro benchmarks [#806](https://github.com/apache/datafusion-comet/pull/806) (andygrove) - Minor: ScanExec code cleanup and additional documentation [#804](https://github.com/apache/datafusion-comet/pull/804) (andygrove) - chore: Make it possible to run 'make benchmark-%' using jvm 17+ [#823](https://github.com/apache/datafusion-comet/pull/823) (eejbyfeldt) - chore: Add more unsupported cases to supportedSortType [#825](https://github.com/apache/datafusion-comet/pull/825) (viirya) - chore: Enable Comet shuffle with AQE coalesce partitions [#834](https://github.com/apache/datafusion-comet/pull/834) (viirya) - chore: Add GitHub workflow to publish Docker image [#847](https://github.com/apache/datafusion-comet/pull/847) (andygrove) - chore: Revert "fix: change the not exists base image apache/spark:3.4.3 to 3.4.2" [#854](https://github.com/apache/datafusion-comet/pull/854) (haoxins) - chore: fix docker-publish attempt 1 [#851](https://github.com/apache/datafusion-comet/pull/851) (andygrove) - minor: stop warning that AQEShuffleRead cannot run natively [#842](https://github.com/apache/datafusion-comet/pull/842) (andygrove) - chore: Improve ObjectHashAggregate fallback error message [#849](https://github.com/apache/datafusion-comet/pull/849) (andygrove) - chore: Fix docker image publishing (specify ghcr.io in tag) [#856](https://github.com/apache/datafusion-comet/pull/856) (andygrove) - chore: Use Git tag as Comet version when publishing Docker images [#857](https://github.com/apache/datafusion-comet/pull/857) (andygrove) ## Credits Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. ``` 36 Andy Grove 16 Liang-Chi Hsieh 9 KAZUYUKI TANIMURA 5 Emil Ejbyfeldt 4 Huaxin Gao 3 Adam Binford 3 Oleks V 3 Xuanwo 2 Arttu 2 Zhen Wang 1 Akhil S S 1 Alexander Ocsa 1 Parth Chandra 1 Xin Hao ``` Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. ================================================ FILE: dev/changelog/0.3.0.md ================================================ # DataFusion Comet 0.3.0 Changelog This release consists of 57 commits from 12 contributors. See credits at the end of this changelog for more information. **Fixed bugs:** - fix: Support type coercion for ScalarUDFs [#865](https://github.com/apache/datafusion-comet/pull/865) (Kimahriman) - fix: CometTakeOrderedAndProjectExec native scan node should use child operator's output [#896](https://github.com/apache/datafusion-comet/pull/896) (viirya) - fix: Fix various memory leaks problems [#890](https://github.com/apache/datafusion-comet/pull/890) (Kontinuation) - fix: Add output to Comet operators equal and hashCode [#902](https://github.com/apache/datafusion-comet/pull/902) (viirya) - fix: Fallback to Spark when cannot resolve AttributeReference [#926](https://github.com/apache/datafusion-comet/pull/926) (viirya) - fix: Fix memory bloat caused by holding too many unclosed `ArrowReaderIterator`s [#929](https://github.com/apache/datafusion-comet/pull/929) (Kontinuation) - fix: Normalize NaN and zeros for floating number comparison [#953](https://github.com/apache/datafusion-comet/pull/953) (viirya) - fix: window function range offset should be long instead of int [#733](https://github.com/apache/datafusion-comet/pull/733) (huaxingao) - fix: CometScanExec on Spark 3.5.2 [#915](https://github.com/apache/datafusion-comet/pull/915) (Kimahriman) - fix: div and rem by negative zero [#960](https://github.com/apache/datafusion-comet/pull/960) (kazuyukitanimura) **Performance related:** - perf: Optimize CometSparkToColumnar for columnar input [#892](https://github.com/apache/datafusion-comet/pull/892) (mbutrovich) - perf: Fall back to Spark if query uses DPP with v1 data sources [#897](https://github.com/apache/datafusion-comet/pull/897) (andygrove) - perf: Report accurate total time for scans [#916](https://github.com/apache/datafusion-comet/pull/916) (andygrove) - perf: Add metric for time spent casting in native scan [#919](https://github.com/apache/datafusion-comet/pull/919) (andygrove) - perf: Add criterion benchmark for aggregate expressions [#948](https://github.com/apache/datafusion-comet/pull/948) (andygrove) - perf: Add metric for time spent in CometSparkToColumnarExec [#931](https://github.com/apache/datafusion-comet/pull/931) (mbutrovich) - perf: Optimize decimal precision check in decimal aggregates (sum and avg) [#952](https://github.com/apache/datafusion-comet/pull/952) (andygrove) **Implemented enhancements:** - feat: Add config option to enable converting CSV to columnar [#871](https://github.com/apache/datafusion-comet/pull/871) (andygrove) - feat: Implement basic version of string to float/double/decimal [#870](https://github.com/apache/datafusion-comet/pull/870) (andygrove) - feat: Implement to_json for subset of types [#805](https://github.com/apache/datafusion-comet/pull/805) (andygrove) - feat: Add ShuffleQueryStageExec to direct child node for CometBroadcastExchangeExec [#880](https://github.com/apache/datafusion-comet/pull/880) (viirya) - feat: Support sort merge join with a join condition [#553](https://github.com/apache/datafusion-comet/pull/553) (viirya) - feat: Array element extraction [#899](https://github.com/apache/datafusion-comet/pull/899) (Kimahriman) - feat: date_add and date_sub functions [#910](https://github.com/apache/datafusion-comet/pull/910) (mbutrovich) - feat: implement scripts for binary release build [#932](https://github.com/apache/datafusion-comet/pull/932) (parthchandra) - feat: Publish artifacts to maven [#946](https://github.com/apache/datafusion-comet/pull/946) (parthchandra) **Documentation updates:** - doc: Documenting Helm chart for Comet Kube execution [#874](https://github.com/apache/datafusion-comet/pull/874) (comphead) - doc: Update native code path in development [#921](https://github.com/apache/datafusion-comet/pull/921) (viirya) - docs: Add more detailed architecture documentation [#922](https://github.com/apache/datafusion-comet/pull/922) (andygrove) **Other:** - chore: Update installation.md [#869](https://github.com/apache/datafusion-comet/pull/869) (haoxins) - chore: Update versions to 0.3.0 / 0.3.0-SNAPSHOT [#868](https://github.com/apache/datafusion-comet/pull/868) (andygrove) - chore: Add documentation on running benchmarks with Microk8s [#848](https://github.com/apache/datafusion-comet/pull/848) (andygrove) - chore: Improve CometExchange metrics [#873](https://github.com/apache/datafusion-comet/pull/873) (viirya) - chore: Add spilling metrics of SortMergeJoin [#878](https://github.com/apache/datafusion-comet/pull/878) (viirya) - chore: change shuffle mode default from jvm to auto [#877](https://github.com/apache/datafusion-comet/pull/877) (andygrove) - chore: Enable shuffle by default [#881](https://github.com/apache/datafusion-comet/pull/881) (andygrove) - chore: print Comet native version to logs after Comet is initialized [#900](https://github.com/apache/datafusion-comet/pull/900) (SemyonSinchenko) - chore: Revise batch pull approach to more follow C Data interface semantics [#893](https://github.com/apache/datafusion-comet/pull/893) (viirya) - chore: Close dictionary provider when iterator is closed [#904](https://github.com/apache/datafusion-comet/pull/904) (andygrove) - chore: Remove unused function [#906](https://github.com/apache/datafusion-comet/pull/906) (viirya) - chore: Upgrade to latest DataFusion revision [#909](https://github.com/apache/datafusion-comet/pull/909) (andygrove) - build: fix build [#917](https://github.com/apache/datafusion-comet/pull/917) (andygrove) - chore: Revise array import to more follow C Data Interface semantics [#905](https://github.com/apache/datafusion-comet/pull/905) (viirya) - chore: Address reviews [#920](https://github.com/apache/datafusion-comet/pull/920) (viirya) - chore: Enable Comet shuffle for Spark core-1 test [#924](https://github.com/apache/datafusion-comet/pull/924) (viirya) - build: Add maven-compiler-plugin for java cross-build [#911](https://github.com/apache/datafusion-comet/pull/911) (viirya) - build: Disable upload-test-reports for macos-13 runner [#933](https://github.com/apache/datafusion-comet/pull/933) (viirya) - minor: cast timestamp test #468 [#923](https://github.com/apache/datafusion-comet/pull/923) (himadripal) - build: Set Java version arg for scala-maven-plugin [#934](https://github.com/apache/datafusion-comet/pull/934) (viirya) - chore: Remove redundant RowToColumnar from CometShuffleExchangeExec for columnar shuffle [#944](https://github.com/apache/datafusion-comet/pull/944) (viirya) - minor: rename CometMetricNode `add` to `set` and update documentation [#940](https://github.com/apache/datafusion-comet/pull/940) (andygrove) - chore: Add config for enabling SMJ with join condition [#937](https://github.com/apache/datafusion-comet/pull/937) (andygrove) - chore: Change maven group ID to `org.apache.datafusion` [#941](https://github.com/apache/datafusion-comet/pull/941) (andygrove) - chore: Upgrade to DataFusion 42.0.0 [#945](https://github.com/apache/datafusion-comet/pull/945) (andygrove) - build: Fix regression in jar packaging [#950](https://github.com/apache/datafusion-comet/pull/950) (andygrove) - chore: Show reason for falling back to Spark when SMJ with join condition is not enabled [#956](https://github.com/apache/datafusion-comet/pull/956) (andygrove) - chore: clarify tarball installation [#959](https://github.com/apache/datafusion-comet/pull/959) (comphead) ## Credits Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. ``` 22 Andy Grove 18 Liang-Chi Hsieh 3 Adam Binford 3 Matt Butrovich 2 Kristin Cowalcijk 2 Oleks V 2 Parth Chandra 1 Himadri Pal 1 Huaxin Gao 1 KAZUYUKI TANIMURA 1 Semyon 1 Xin Hao ``` Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. ================================================ FILE: dev/changelog/0.4.0.md ================================================ # DataFusion Comet 0.4.0 Changelog This release consists of 51 commits from 10 contributors. See credits at the end of this changelog for more information. **Fixed bugs:** - fix: Use the number of rows from underlying arrays instead of logical row count from RecordBatch [#972](https://github.com/apache/datafusion-comet/pull/972) (viirya) - fix: The spilled_bytes metric of CometSortExec should be size instead of time [#984](https://github.com/apache/datafusion-comet/pull/984) (Kontinuation) - fix: Properly handle Java exceptions without error messages; fix loading of comet native library from java.library.path [#982](https://github.com/apache/datafusion-comet/pull/982) (Kontinuation) - fix: Fallback to Spark if scan has meta columns [#997](https://github.com/apache/datafusion-comet/pull/997) (viirya) - fix: Fallback to Spark if named_struct contains duplicate field names [#1016](https://github.com/apache/datafusion-comet/pull/1016) (viirya) - fix: Make comet-git-info.properties optional [#1027](https://github.com/apache/datafusion-comet/pull/1027) (andygrove) - fix: TopK operator should return correct results on dictionary column with nulls [#1033](https://github.com/apache/datafusion-comet/pull/1033) (viirya) - fix: need default value for getSizeAsMb(EXECUTOR_MEMORY.key) [#1046](https://github.com/apache/datafusion-comet/pull/1046) (neyama) **Performance related:** - perf: Remove one redundant CopyExec for SMJ [#962](https://github.com/apache/datafusion-comet/pull/962) (andygrove) - perf: Add experimental feature to replace SortMergeJoin with ShuffledHashJoin [#1007](https://github.com/apache/datafusion-comet/pull/1007) (andygrove) - perf: Cache jstrings during metrics collection [#1029](https://github.com/apache/datafusion-comet/pull/1029) (mbutrovich) **Implemented enhancements:** - feat: Support `GetArrayStructFields` expression [#993](https://github.com/apache/datafusion-comet/pull/993) (Kimahriman) - feat: Implement bloom_filter_agg [#987](https://github.com/apache/datafusion-comet/pull/987) (mbutrovich) - feat: Support more types with BloomFilterAgg [#1039](https://github.com/apache/datafusion-comet/pull/1039) (mbutrovich) - feat: Implement CAST from struct to string [#1066](https://github.com/apache/datafusion-comet/pull/1066) (andygrove) - feat: Use official DataFusion 43 release [#1070](https://github.com/apache/datafusion-comet/pull/1070) (andygrove) - feat: Implement CAST between struct types [#1074](https://github.com/apache/datafusion-comet/pull/1074) (andygrove) - feat: support array_append [#1072](https://github.com/apache/datafusion-comet/pull/1072) (NoeB) - feat: Require offHeap memory to be enabled (always use unified memory) [#1062](https://github.com/apache/datafusion-comet/pull/1062) (andygrove) **Documentation updates:** - doc: add documentation interlinks [#975](https://github.com/apache/datafusion-comet/pull/975) (comphead) - docs: Add IntelliJ documentation for generated source code [#985](https://github.com/apache/datafusion-comet/pull/985) (mbutrovich) - docs: Update tuning guide [#995](https://github.com/apache/datafusion-comet/pull/995) (andygrove) - docs: Various documentation improvements [#1005](https://github.com/apache/datafusion-comet/pull/1005) (andygrove) - docs: clarify that Maven central only has jars for Linux [#1009](https://github.com/apache/datafusion-comet/pull/1009) (andygrove) - doc: fix K8s links and doc [#1058](https://github.com/apache/datafusion-comet/pull/1058) (comphead) - docs: Update benchmarking.md [#1085](https://github.com/apache/datafusion-comet/pull/1085) (rluvaton-flarion) **Other:** - chore: Generate changelog for 0.3.0 release [#964](https://github.com/apache/datafusion-comet/pull/964) (andygrove) - chore: fix publish-to-maven script [#966](https://github.com/apache/datafusion-comet/pull/966) (andygrove) - chore: Update benchmarks results based on 0.3.0-rc1 [#969](https://github.com/apache/datafusion-comet/pull/969) (andygrove) - chore: update rem expression guide [#976](https://github.com/apache/datafusion-comet/pull/976) (kazuyukitanimura) - chore: Enable additional CreateArray tests [#928](https://github.com/apache/datafusion-comet/pull/928) (Kimahriman) - chore: fix compatibility guide [#978](https://github.com/apache/datafusion-comet/pull/978) (kazuyukitanimura) - chore: Update for 0.3.0 release, prepare for 0.4.0 development [#970](https://github.com/apache/datafusion-comet/pull/970) (andygrove) - chore: Don't transform the HashAggregate to CometHashAggregate if Comet shuffle is disabled [#991](https://github.com/apache/datafusion-comet/pull/991) (viirya) - chore: Make parquet reader options Comet options instead of Hadoop options [#968](https://github.com/apache/datafusion-comet/pull/968) (parthchandra) - chore: remove legacy comet-spark-shell [#1013](https://github.com/apache/datafusion-comet/pull/1013) (andygrove) - chore: Reserve memory for native shuffle writer per partition [#988](https://github.com/apache/datafusion-comet/pull/988) (viirya) - chore: Bump arrow-rs to 53.1.0 and datafusion [#1001](https://github.com/apache/datafusion-comet/pull/1001) (kazuyukitanimura) - chore: Revert "chore: Reserve memory for native shuffle writer per partition (#988)" [#1020](https://github.com/apache/datafusion-comet/pull/1020) (viirya) - minor: Remove hard-coded version number from Dockerfile [#1025](https://github.com/apache/datafusion-comet/pull/1025) (andygrove) - chore: Reserve memory for native shuffle writer per partition [#1022](https://github.com/apache/datafusion-comet/pull/1022) (viirya) - chore: Improve error handling when native lib fails to load [#1000](https://github.com/apache/datafusion-comet/pull/1000) (andygrove) - chore: Use twox-hash 2.0 xxhash64 oneshot api instead of custom implementation [#1041](https://github.com/apache/datafusion-comet/pull/1041) (NoeB) - chore: Refactor Arrow Array and Schema allocation in ColumnReader and MetadataColumnReader [#1047](https://github.com/apache/datafusion-comet/pull/1047) (viirya) - minor: Refactor binary expr serde to reduce code duplication [#1053](https://github.com/apache/datafusion-comet/pull/1053) (andygrove) - chore: Upgrade to DataFusion 43.0.0-rc1 [#1057](https://github.com/apache/datafusion-comet/pull/1057) (andygrove) - chore: Refactor UnaryExpr and MathExpr in protobuf [#1056](https://github.com/apache/datafusion-comet/pull/1056) (andygrove) - minor: use defaults instead of hard-coding values [#1060](https://github.com/apache/datafusion-comet/pull/1060) (andygrove) - minor: refactor UnaryExpr handling to make code more concise [#1065](https://github.com/apache/datafusion-comet/pull/1065) (andygrove) - chore: Refactor binary and math expression serde code [#1069](https://github.com/apache/datafusion-comet/pull/1069) (andygrove) - chore: Simplify CometShuffleMemoryAllocator to use Spark unified memory allocator [#1063](https://github.com/apache/datafusion-comet/pull/1063) (viirya) - test: Restore one test in CometExecSuite by adding COMET_SHUFFLE_MODE config [#1087](https://github.com/apache/datafusion-comet/pull/1087) (viirya) ## Credits Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. ``` 19 Andy Grove 13 Matt Butrovich 8 Liang-Chi Hsieh 3 KAZUYUKI TANIMURA 2 Adam Binford 2 Kristin Cowalcijk 1 NoeB 1 Oleks V 1 Parth Chandra 1 neyama ``` Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. ================================================ FILE: dev/changelog/0.5.0.md ================================================ # DataFusion Comet 0.5.0 Changelog This release consists of 69 commits from 15 contributors. See credits at the end of this changelog for more information. **Fixed bugs:** - fix: Unsigned type related bugs [#1095](https://github.com/apache/datafusion-comet/pull/1095) (kazuyukitanimura) - fix: Use RDD partition index [#1112](https://github.com/apache/datafusion-comet/pull/1112) (viirya) - fix: Various metrics bug fixes and improvements [#1111](https://github.com/apache/datafusion-comet/pull/1111) (andygrove) - fix: Don't create CometScanExec for subclasses of ParquetFileFormat [#1129](https://github.com/apache/datafusion-comet/pull/1129) (Kimahriman) - fix: Fix metrics regressions [#1132](https://github.com/apache/datafusion-comet/pull/1132) (andygrove) - fix: Enable scenarios accidentally commented out in CometExecBenchmark [#1151](https://github.com/apache/datafusion-comet/pull/1151) (mbutrovich) - fix: Spark 4.0-preview1 SPARK-47120 [#1156](https://github.com/apache/datafusion-comet/pull/1156) (kazuyukitanimura) - fix: Document enabling comet explain plan usage in Spark (4.0) [#1176](https://github.com/apache/datafusion-comet/pull/1176) (parthchandra) - fix: stddev_pop should not directly return 0.0 when count is 1.0 [#1184](https://github.com/apache/datafusion-comet/pull/1184) (viirya) - fix: fix missing explanation for then branch in case when [#1200](https://github.com/apache/datafusion-comet/pull/1200) (rluvaton) - fix: Fall back to Spark for unsupported partition or sort expressions in window aggregates [#1253](https://github.com/apache/datafusion-comet/pull/1253) (andygrove) - fix: Fall back to Spark for distinct aggregates [#1262](https://github.com/apache/datafusion-comet/pull/1262) (andygrove) - fix: disable initCap by default [#1276](https://github.com/apache/datafusion-comet/pull/1276) (kazuyukitanimura) **Performance related:** - perf: Stop passing Java config map into native createPlan [#1101](https://github.com/apache/datafusion-comet/pull/1101) (andygrove) - feat: Make native shuffle compression configurable and respect `spark.shuffle.compress` [#1185](https://github.com/apache/datafusion-comet/pull/1185) (andygrove) - perf: Improve query planning to more reliably fall back to columnar shuffle when native shuffle is not supported [#1209](https://github.com/apache/datafusion-comet/pull/1209) (andygrove) - feat: Move shuffle block decompression and decoding to native code and add LZ4 & Snappy support [#1192](https://github.com/apache/datafusion-comet/pull/1192) (andygrove) - feat: Implement custom RecordBatch serde for shuffle for improved performance [#1190](https://github.com/apache/datafusion-comet/pull/1190) (andygrove) **Implemented enhancements:** - feat: support array_insert [#1073](https://github.com/apache/datafusion-comet/pull/1073) (SemyonSinchenko) - feat: enable decimal to decimal cast of different precision and scale [#1086](https://github.com/apache/datafusion-comet/pull/1086) (himadripal) - feat: Improve ScanExec native metrics [#1133](https://github.com/apache/datafusion-comet/pull/1133) (andygrove) - feat: Add Spark-compatible implementation of SchemaAdapterFactory [#1169](https://github.com/apache/datafusion-comet/pull/1169) (andygrove) - feat: Improve shuffle metrics (second attempt) [#1175](https://github.com/apache/datafusion-comet/pull/1175) (andygrove) - feat: Add a `spark.comet.exec.memoryPool` configuration for experimenting with various datafusion memory pool setups. [#1021](https://github.com/apache/datafusion-comet/pull/1021) (Kontinuation) - feat: Reenable tests for filtered SMJ anti join [#1211](https://github.com/apache/datafusion-comet/pull/1211) (comphead) - feat: add support for array_remove expression [#1179](https://github.com/apache/datafusion-comet/pull/1179) (jatin510) **Documentation updates:** - docs: Update documentation for 0.4.0 release [#1096](https://github.com/apache/datafusion-comet/pull/1096) (andygrove) - docs: Fix readme typo FGPA -> FPGA [#1117](https://github.com/apache/datafusion-comet/pull/1117) (gstvg) - docs: Add more technical detail and new diagram to Comet plugin overview [#1119](https://github.com/apache/datafusion-comet/pull/1119) (andygrove) - docs: Add some documentation explaining how shuffle works [#1148](https://github.com/apache/datafusion-comet/pull/1148) (andygrove) - docs: Update TPC-H benchmark results [#1257](https://github.com/apache/datafusion-comet/pull/1257) (andygrove) **Other:** - chore: Add changelog for 0.4.0 [#1089](https://github.com/apache/datafusion-comet/pull/1089) (andygrove) - chore: Prepare for 0.5.0 development [#1090](https://github.com/apache/datafusion-comet/pull/1090) (andygrove) - build: Skip installation of spark-integration and fuzz testing modules [#1091](https://github.com/apache/datafusion-comet/pull/1091) (parthchandra) - minor: Add hint for finding the GPG key to use when publishing to maven [#1093](https://github.com/apache/datafusion-comet/pull/1093) (andygrove) - chore: Include first ScanExec batch in metrics [#1105](https://github.com/apache/datafusion-comet/pull/1105) (andygrove) - chore: Improve CometScan metrics [#1100](https://github.com/apache/datafusion-comet/pull/1100) (andygrove) - chore: Add custom metric for native shuffle fetching batches from JVM [#1108](https://github.com/apache/datafusion-comet/pull/1108) (andygrove) - chore: Remove unused StringView struct [#1143](https://github.com/apache/datafusion-comet/pull/1143) (andygrove) - test: enable more Spark 4.0 tests [#1145](https://github.com/apache/datafusion-comet/pull/1145) (kazuyukitanimura) - chore: Refactor cast to use SparkCastOptions param [#1146](https://github.com/apache/datafusion-comet/pull/1146) (andygrove) - chore: Move more expressions from core crate to spark-expr crate [#1152](https://github.com/apache/datafusion-comet/pull/1152) (andygrove) - chore: Remove dead code [#1155](https://github.com/apache/datafusion-comet/pull/1155) (andygrove) - chore: Move string kernels and expressions to spark-expr crate [#1164](https://github.com/apache/datafusion-comet/pull/1164) (andygrove) - chore: Move remaining expressions to spark-expr crate + some minor refactoring [#1165](https://github.com/apache/datafusion-comet/pull/1165) (andygrove) - chore: Add ignored tests for reading complex types from Parquet [#1167](https://github.com/apache/datafusion-comet/pull/1167) (andygrove) - test: enabling Spark tests with offHeap requirement [#1177](https://github.com/apache/datafusion-comet/pull/1177) (kazuyukitanimura) - minor: move shuffle classes from common to spark [#1193](https://github.com/apache/datafusion-comet/pull/1193) (andygrove) - minor: refactor to move decodeBatches to broadcast exchange code as private function [#1195](https://github.com/apache/datafusion-comet/pull/1195) (andygrove) - minor: refactor prepare_output so that it does not require an ExecutionContext [#1194](https://github.com/apache/datafusion-comet/pull/1194) (andygrove) - minor: remove unused source files [#1202](https://github.com/apache/datafusion-comet/pull/1202) (andygrove) - chore: Upgrade to DataFusion 44.0.0-rc2 [#1154](https://github.com/apache/datafusion-comet/pull/1154) (andygrove) - chore: Add safety check to CometBuffer [#1050](https://github.com/apache/datafusion-comet/pull/1050) (viirya) - chore: Remove unreachable code [#1213](https://github.com/apache/datafusion-comet/pull/1213) (andygrove) - test: Enable Comet by default except some tests in SparkSessionExtensionSuite [#1201](https://github.com/apache/datafusion-comet/pull/1201) (kazuyukitanimura) - chore: extract `struct` expressions to folders based on spark grouping [#1216](https://github.com/apache/datafusion-comet/pull/1216) (rluvaton) - chore: extract static invoke expressions to folders based on spark grouping [#1217](https://github.com/apache/datafusion-comet/pull/1217) (rluvaton) - chore: Follow-on PR to fully enable onheap memory usage [#1210](https://github.com/apache/datafusion-comet/pull/1210) (andygrove) - chore: extract agg_funcs expressions to folders based on spark grouping [#1224](https://github.com/apache/datafusion-comet/pull/1224) (rluvaton) - chore: extract datetime_funcs expressions to folders based on spark grouping [#1222](https://github.com/apache/datafusion-comet/pull/1222) (rluvaton) - chore: Upgrade to DataFusion 44.0.0 from 44.0.0 RC2 [#1232](https://github.com/apache/datafusion-comet/pull/1232) (rluvaton) - chore: extract strings file to `strings_func` like in spark grouping [#1215](https://github.com/apache/datafusion-comet/pull/1215) (rluvaton) - chore: extract predicate_functions expressions to folders based on spark grouping [#1218](https://github.com/apache/datafusion-comet/pull/1218) (rluvaton) - build(deps): bump protobuf version to 3.21.12 [#1234](https://github.com/apache/datafusion-comet/pull/1234) (wForget) - chore: extract json_funcs expressions to folders based on spark grouping [#1220](https://github.com/apache/datafusion-comet/pull/1220) (rluvaton) - test: Enable shuffle by default in Spark tests [#1240](https://github.com/apache/datafusion-comet/pull/1240) (kazuyukitanimura) - chore: extract hash_funcs expressions to folders based on spark grouping [#1221](https://github.com/apache/datafusion-comet/pull/1221) (rluvaton) - build: Fix test failure caused by merging conflicting PRs [#1259](https://github.com/apache/datafusion-comet/pull/1259) (andygrove) ## Credits Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. ``` 37 Andy Grove 10 Raz Luvaton 7 KAZUYUKI TANIMURA 3 Liang-Chi Hsieh 2 Parth Chandra 1 Adam Binford 1 Dharan Aditya 1 Himadri Pal 1 Jagdish Parihar 1 Kristin Cowalcijk 1 Matt Butrovich 1 Oleks V 1 Sem 1 Zhen Wang 1 gstvg ``` Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. ================================================ FILE: dev/changelog/0.6.0.md ================================================ # DataFusion Comet 0.6.0 Changelog **Fixed bugs:** - fix: cast timestamp to decimal is unsupported [#1281](https://github.com/apache/datafusion-comet/pull/1281) (wForget) - fix: partially fix consistency issue of hash functions with decimal input [#1295](https://github.com/apache/datafusion-comet/pull/1295) (wForget) - fix: Improve testing for array_remove and fallback to Spark for unsupported types [#1308](https://github.com/apache/datafusion-comet/pull/1308) (andygrove) - fix: address post merge comet-parquet-exec review comments [#1327](https://github.com/apache/datafusion-comet/pull/1327) (parthchandra) - fix: memory pool error type [#1346](https://github.com/apache/datafusion-comet/pull/1346) (kazuyukitanimura) - fix: Fall back to Spark when hashing decimals with precision > 18 [#1325](https://github.com/apache/datafusion-comet/pull/1325) (andygrove) - fix: expressions doc for ArrayRemove [#1356](https://github.com/apache/datafusion-comet/pull/1356) (kazuyukitanimura) - fix: pass scale to DF round in spark_round [#1341](https://github.com/apache/datafusion-comet/pull/1341) (cht42) - fix: Mark cast from float/double to decimal as incompatible [#1372](https://github.com/apache/datafusion-comet/pull/1372) (andygrove) - fix: Passthrough condition in StaticInvoke case block [#1392](https://github.com/apache/datafusion-comet/pull/1392) (EmilyMatt) - fix: disable checking for uint_8 and uint_16 if complex type readers are enabled [#1376](https://github.com/apache/datafusion-comet/pull/1376) (parthchandra) **Performance related:** - perf: improve performance of update metrics [#1329](https://github.com/apache/datafusion-comet/pull/1329) (wForget) - perf: Use DataFusion FilterExec for experimental native scans [#1395](https://github.com/apache/datafusion-comet/pull/1395) (mbutrovich) **Implemented enhancements:** - feat: Add HasRowIdMapping interface [#1288](https://github.com/apache/datafusion-comet/pull/1288) (viirya) - feat: Upgrade to DataFusion 45 [#1364](https://github.com/apache/datafusion-comet/pull/1364) (andygrove) - feat: Add fair unified memory pool [#1369](https://github.com/apache/datafusion-comet/pull/1369) (kazuyukitanimura) - feat: Add unbounded memory pool [#1386](https://github.com/apache/datafusion-comet/pull/1386) (kazuyukitanimura) - feat: make random seed configurable in fuzz-testing [#1401](https://github.com/apache/datafusion-comet/pull/1401) (wForget) - feat: override executor overhead memory only when comet unified memory manager is disabled [#1379](https://github.com/apache/datafusion-comet/pull/1379) (wForget) **Documentation updates:** - docs: Fix links and provide complete benchmarking scripts [#1284](https://github.com/apache/datafusion-comet/pull/1284) (andygrove) - doc: update memory tuning guide [#1394](https://github.com/apache/datafusion-comet/pull/1394) (kazuyukitanimura) **Other:** - chore: Start 0.6.0 development [#1286](https://github.com/apache/datafusion-comet/pull/1286) (andygrove) - minor: update compatibility [#1303](https://github.com/apache/datafusion-comet/pull/1303) (kazuyukitanimura) - chore: extract conversion_funcs, conditional_funcs, bitwise_funcs and array_funcs expressions to folders based on spark grouping [#1223](https://github.com/apache/datafusion-comet/pull/1223) (rluvaton) - chore: extract math_funcs expressions to folders based on spark grouping [#1219](https://github.com/apache/datafusion-comet/pull/1219) (rluvaton) - chore: merge comet-parquet-exec branch into main [#1318](https://github.com/apache/datafusion-comet/pull/1318) (andygrove) - Feat: Support array_intersect function [#1271](https://github.com/apache/datafusion-comet/pull/1271) (erenavsarogullari) - build(deps): bump pprof from 0.13.0 to 0.14.0 in /native [#1319](https://github.com/apache/datafusion-comet/pull/1319) (dependabot[bot]) - chore: Fix merge conflicts from merging comet-parquet-exec into main [#1320](https://github.com/apache/datafusion-comet/pull/1320) (andygrove) - chore: Revert accidental re-introduction of off-heap memory requirement [#1326](https://github.com/apache/datafusion-comet/pull/1326) (andygrove) - chore: Fix merge conflicts from merging comet-parquet-exec into main [#1323](https://github.com/apache/datafusion-comet/pull/1323) (mbutrovich) - Feat: Support array_join function [#1290](https://github.com/apache/datafusion-comet/pull/1290) (erenavsarogullari) - Fix missing slash in spark script [#1334](https://github.com/apache/datafusion-comet/pull/1334) (xleoken) - chore: Refactor QueryPlanSerde to allow logic to be moved to individual classes per expression [#1331](https://github.com/apache/datafusion-comet/pull/1331) (andygrove) - build: re-enable upload-test-reports for macos-13 runner [#1335](https://github.com/apache/datafusion-comet/pull/1335) (viirya) - chore: Upgrade to Arrow 53.4.0 [#1338](https://github.com/apache/datafusion-comet/pull/1338) (andygrove) - Feat: Support arrays_overlap function [#1312](https://github.com/apache/datafusion-comet/pull/1312) (erenavsarogullari) - chore: Move all array\_\* serde to new framework, use correct INCOMPAT config [#1349](https://github.com/apache/datafusion-comet/pull/1349) (andygrove) - chore: Prepare for DataFusion 45 (bump to DataFusion rev 5592834 + Arrow 54.0.0) [#1332](https://github.com/apache/datafusion-comet/pull/1332) (andygrove) - minor: commit compatibility doc [#1358](https://github.com/apache/datafusion-comet/pull/1358) (kazuyukitanimura) - minor: update fuzz dependency [#1357](https://github.com/apache/datafusion-comet/pull/1357) (kazuyukitanimura) - chore: Remove redundant processing from exprToProtoInternal [#1351](https://github.com/apache/datafusion-comet/pull/1351) (andygrove) - chore: Adding an optional `hdfs` crate [#1377](https://github.com/apache/datafusion-comet/pull/1377) (comphead) - chore: Refactor aggregate expression serde [#1380](https://github.com/apache/datafusion-comet/pull/1380) (andygrove) ================================================ FILE: dev/changelog/0.7.0.md ================================================ # DataFusion Comet 0.7.0 Changelog This release consists of 46 commits from 11 contributors. See credits at the end of this changelog for more information. **Fixed bugs:** - fix: Change default value of COMET_SCAN_ALLOW_INCOMPATIBLE and add documentation [#1398](https://github.com/apache/datafusion-comet/pull/1398) (andygrove) - fix: Reduce cast.rs and utils.rs logic from parquet_support.rs for experimental native scans [#1387](https://github.com/apache/datafusion-comet/pull/1387) (mbutrovich) - fix: Remove more cast.rs logic from parquet_support.rs for experimental native scans [#1413](https://github.com/apache/datafusion-comet/pull/1413) (mbutrovich) - fix: fix various unit test failures in native_datafusion and native_iceberg_compat readers [#1415](https://github.com/apache/datafusion-comet/pull/1415) (parthchandra) - fix: metrics tests for native_datafusion experimental native scan [#1445](https://github.com/apache/datafusion-comet/pull/1445) (mbutrovich) - fix: Reduce number of shuffle spill files, fix spilled_bytes metric, add some unit tests [#1440](https://github.com/apache/datafusion-comet/pull/1440) (andygrove) - fix: Executor memory overhead overriding [#1462](https://github.com/apache/datafusion-comet/pull/1462) (LukMRVC) - fix: Stop copying rust-toolchain to docker file [#1475](https://github.com/apache/datafusion-comet/pull/1475) (andygrove) - fix: PartitionBuffers should not have their own MemoryConsumer [#1496](https://github.com/apache/datafusion-comet/pull/1496) (EmilyMatt) - fix: enable full decimal to decimal support [#1385](https://github.com/apache/datafusion-comet/pull/1385) (himadripal) - fix: use common implementation of handling object store and hdfs urls for native_datafusion and native_iceberg_compat [#1494](https://github.com/apache/datafusion-comet/pull/1494) (parthchandra) - fix: Simplify CometShuffleMemoryAllocator logic, rename classes, remove config [#1485](https://github.com/apache/datafusion-comet/pull/1485) (mbutrovich) - fix: check overflow for decimal integral division [#1512](https://github.com/apache/datafusion-comet/pull/1512) (wForget) **Performance related:** - perf: Update RewriteJoin logic to choose optimal build side [#1424](https://github.com/apache/datafusion-comet/pull/1424) (andygrove) - perf: Reduce native shuffle memory overhead by 50% [#1452](https://github.com/apache/datafusion-comet/pull/1452) (andygrove) **Implemented enhancements:** - feat: CometNativeScan metrics from ParquetFileMetrics and FileStreamMetrics [#1172](https://github.com/apache/datafusion-comet/pull/1172) (mbutrovich) - feat: add experimental remote HDFS support for native DataFusion reader [#1359](https://github.com/apache/datafusion-comet/pull/1359) (comphead) - feat: add Win-amd64 profile [#1410](https://github.com/apache/datafusion-comet/pull/1410) (wForget) - feat: Support IntegralDivide function [#1428](https://github.com/apache/datafusion-comet/pull/1428) (wForget) - feat: Add div operator for fuzz testing and update expression doc [#1464](https://github.com/apache/datafusion-comet/pull/1464) (wForget) - feat: Upgrade to DataFusion 46.0.0-rc2 [#1423](https://github.com/apache/datafusion-comet/pull/1423) (andygrove) - feat: Add support for rpad [#1470](https://github.com/apache/datafusion-comet/pull/1470) (andygrove) - feat: Use official DataFusion 46.0.0 release [#1484](https://github.com/apache/datafusion-comet/pull/1484) (andygrove) **Documentation updates:** - docs: Add changelog for 0.6.0 release [#1402](https://github.com/apache/datafusion-comet/pull/1402) (andygrove) - docs: Improve documentation for running stability plan tests [#1469](https://github.com/apache/datafusion-comet/pull/1469) (andygrove) **Other:** - test: Add experimental native scans to CometReadBenchmark [#1150](https://github.com/apache/datafusion-comet/pull/1150) (mbutrovich) - chore: Prepare for 0.7.0 development [#1404](https://github.com/apache/datafusion-comet/pull/1404) (andygrove) - chore: Update released version in documentation [#1418](https://github.com/apache/datafusion-comet/pull/1418) (andygrove) - chore: Update protobuf to 3.25.5 [#1434](https://github.com/apache/datafusion-comet/pull/1434) (kazuyukitanimura) - chore: Update guava to 33.2.1-jre [#1435](https://github.com/apache/datafusion-comet/pull/1435) (kazuyukitanimura) - test: Register Spark-compatible expressions with a DataFusion context [#1432](https://github.com/apache/datafusion-comet/pull/1432) (viczsaurav) - chore: fixes for kube build [#1421](https://github.com/apache/datafusion-comet/pull/1421) (comphead) - build: pin machete to version 0.7.0 [#1444](https://github.com/apache/datafusion-comet/pull/1444) (andygrove) - chore: Re-organize shuffle writer code [#1439](https://github.com/apache/datafusion-comet/pull/1439) (andygrove) - chore: faster maven mirror [#1447](https://github.com/apache/datafusion-comet/pull/1447) (comphead) - build: Use stable channel in rust-toolchain [#1465](https://github.com/apache/datafusion-comet/pull/1465) (andygrove) - Feat: support array_compact function [#1321](https://github.com/apache/datafusion-comet/pull/1321) (kazantsev-maksim) - chore: Upgrade to Spark 3.5.4 [#1471](https://github.com/apache/datafusion-comet/pull/1471) (andygrove) - chore: Enable CI checks for `native_datafusion` scan [#1479](https://github.com/apache/datafusion-comet/pull/1479) (andygrove) - chore: Add `native_iceberg_compat` CI checks [#1487](https://github.com/apache/datafusion-comet/pull/1487) (andygrove) - chore: Stop disabling readside padding in TPC stability suite [#1491](https://github.com/apache/datafusion-comet/pull/1491) (andygrove) - chore: Remove num partitions from repartitioner [#1498](https://github.com/apache/datafusion-comet/pull/1498) (EmilyMatt) - test: fix Spark 3.5 tests [#1482](https://github.com/apache/datafusion-comet/pull/1482) (kazuyukitanimura) - minor: Remove hard-coded config default [#1503](https://github.com/apache/datafusion-comet/pull/1503) (andygrove) - chore: Use Datafusion's existing empty stream [#1517](https://github.com/apache/datafusion-comet/pull/1517) (EmilyMatt) ## Credits Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. ``` 20 Andy Grove 6 Matt Butrovich 4 Zhen Wang 3 Emily Matheys 3 KAZUYUKI TANIMURA 3 Oleks V 2 Himadri Pal 2 Parth Chandra 1 Kazantsev Maksim 1 Lukas Moravec 1 Saurav Verma ``` Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. ================================================ FILE: dev/changelog/0.8.0.md ================================================ # DataFusion Comet 0.8.0 Changelog This release consists of 81 commits from 11 contributors. See credits at the end of this changelog for more information. **Fixed bugs:** - fix: remove code duplication in native_datafusion and native_iceberg_compat implementations [#1443](https://github.com/apache/datafusion-comet/pull/1443) (parthchandra) - fix: Refactor CometScanRule and fix bugs [#1483](https://github.com/apache/datafusion-comet/pull/1483) (andygrove) - fix: check if handle has been initialized before closing [#1554](https://github.com/apache/datafusion-comet/pull/1554) (wForget) - fix: Taking slicing into account when writing BooleanBuffers as fast-encoding format [#1522](https://github.com/apache/datafusion-comet/pull/1522) (Kontinuation) - fix: isCometEnabled name conflict [#1569](https://github.com/apache/datafusion-comet/pull/1569) (kazuyukitanimura) - fix: make register_object_store use same session_env as file scan [#1555](https://github.com/apache/datafusion-comet/pull/1555) (wForget) - fix: adjust CometNativeScan's doCanonicalize and hashCode for AQE, use DataSourceScanExec trait [#1578](https://github.com/apache/datafusion-comet/pull/1578) (mbutrovich) - fix: corrected the logic of eliminating CometSparkToColumnarExec [#1597](https://github.com/apache/datafusion-comet/pull/1597) (wForget) - fix: avoid panic caused by close null handle of parquet reader [#1604](https://github.com/apache/datafusion-comet/pull/1604) (wForget) - fix: Make AQE capable of converting Comet shuffled joins to Comet broadcast hash joins [#1605](https://github.com/apache/datafusion-comet/pull/1605) (Kontinuation) - fix: Making shuffle files generated in native shuffle mode reclaimable [#1568](https://github.com/apache/datafusion-comet/pull/1568) (Kontinuation) - fix: Support per-task shuffle write rows and shuffle write time metrics [#1617](https://github.com/apache/datafusion-comet/pull/1617) (Kontinuation) - fix: Modify Spark SQL core 2 tests for `native_datafusion` reader, change 3.5.5 diff hash length to 11 [#1641](https://github.com/apache/datafusion-comet/pull/1641) (mbutrovich) - fix: fix spark/sql test failures in native_iceberg_compat [#1593](https://github.com/apache/datafusion-comet/pull/1593) (parthchandra) - fix: handle missing field correctly in native_iceberg_compat [#1656](https://github.com/apache/datafusion-comet/pull/1656) (parthchandra) - fix: better int96 support for experimental native scans [#1652](https://github.com/apache/datafusion-comet/pull/1652) (mbutrovich) - fix: respect `ignoreNulls` flag in `first_value` and `last_value` [#1626](https://github.com/apache/datafusion-comet/pull/1626) (andygrove) - fix: update row groups count in internal metrics accumulator [#1658](https://github.com/apache/datafusion-comet/pull/1658) (parthchandra) - fix: Shuffle should maintain insertion order [#1660](https://github.com/apache/datafusion-comet/pull/1660) (EmilyMatt) **Performance related:** - perf: Use a global tokio runtime [#1614](https://github.com/apache/datafusion-comet/pull/1614) (andygrove) - perf: Respect Spark's PARQUET_FILTER_PUSHDOWN_ENABLED config [#1619](https://github.com/apache/datafusion-comet/pull/1619) (andygrove) - perf: Experimental fix to avoid join strategy regression [#1674](https://github.com/apache/datafusion-comet/pull/1674) (andygrove) **Implemented enhancements:** - feat: add read array support [#1456](https://github.com/apache/datafusion-comet/pull/1456) (comphead) - feat: introduce hadoop mini cluster to test native scan on hdfs [#1556](https://github.com/apache/datafusion-comet/pull/1556) (wForget) - feat: make parquet native scan schema case insensitive [#1575](https://github.com/apache/datafusion-comet/pull/1575) (wForget) - feat: enable iceberg compat tests, more tests for complex types [#1550](https://github.com/apache/datafusion-comet/pull/1550) (comphead) - feat: pushdown filter for native_iceberg_compat [#1566](https://github.com/apache/datafusion-comet/pull/1566) (wForget) - feat: Fix struct of arrays schema issue [#1592](https://github.com/apache/datafusion-comet/pull/1592) (comphead) - feat: adding more struct/arrays tests [#1594](https://github.com/apache/datafusion-comet/pull/1594) (comphead) - feat: respect `batchSize/workerThreads/blockingThreads` configurations for native_iceberg_compat scan [#1587](https://github.com/apache/datafusion-comet/pull/1587) (wForget) - feat: add MAP type support for first level [#1603](https://github.com/apache/datafusion-comet/pull/1603) (comphead) - feat: Add more tests for nested types combinations for `native_datafusion` [#1632](https://github.com/apache/datafusion-comet/pull/1632) (comphead) - feat: Override MapBuilder values field with expected schema [#1643](https://github.com/apache/datafusion-comet/pull/1643) (comphead) - feat: track unified memory pool [#1651](https://github.com/apache/datafusion-comet/pull/1651) (wForget) - feat: Add support for complex types in native shuffle [#1655](https://github.com/apache/datafusion-comet/pull/1655) (andygrove) **Documentation updates:** - docs: Update configuration guide to show optional configs [#1524](https://github.com/apache/datafusion-comet/pull/1524) (andygrove) - docs: Add changelog for 0.7.0 release [#1527](https://github.com/apache/datafusion-comet/pull/1527) (andygrove) - docs: Use a shallow clone for Spark SQL test instructions [#1547](https://github.com/apache/datafusion-comet/pull/1547) (mbutrovich) - docs: Update benchmark results for 0.7.0 release [#1548](https://github.com/apache/datafusion-comet/pull/1548) (andygrove) - doc: Renew `kubernetes.md` [#1549](https://github.com/apache/datafusion-comet/pull/1549) (comphead) - docs: various improvements to tuning guide [#1525](https://github.com/apache/datafusion-comet/pull/1525) (andygrove) - docs: Update supported Spark versions [#1580](https://github.com/apache/datafusion-comet/pull/1580) (andygrove) - docs: change OSX/OS X to macOS [#1584](https://github.com/apache/datafusion-comet/pull/1584) (mbutrovich) - docs: docs for benchmarking in aws ec2 [#1601](https://github.com/apache/datafusion-comet/pull/1601) (andygrove) - docs: Update compatibility docs for new native scans [#1657](https://github.com/apache/datafusion-comet/pull/1657) (andygrove) - doc: Document local HDFS setup [#1673](https://github.com/apache/datafusion-comet/pull/1673) (comphead) **Other:** - chore: fix issue in release process [#1528](https://github.com/apache/datafusion-comet/pull/1528) (andygrove) - chore: Remove all subdependencies [#1514](https://github.com/apache/datafusion-comet/pull/1514) (EmilyMatt) - chore: Drop support for Spark 3.3 (EOL) [#1529](https://github.com/apache/datafusion-comet/pull/1529) (andygrove) - chore: Prepare for 0.8.0 development [#1530](https://github.com/apache/datafusion-comet/pull/1530) (andygrove) - chore: Re-enable GitHub discussions [#1535](https://github.com/apache/datafusion-comet/pull/1535) (andygrove) - chore: [FOLLOWUP] Drop support for Spark 3.3 (EOL) [#1534](https://github.com/apache/datafusion-comet/pull/1534) (kazuyukitanimura) - build: Use unique name for surefire artifacts [#1544](https://github.com/apache/datafusion-comet/pull/1544) (andygrove) - chore: Update links for released version [#1540](https://github.com/apache/datafusion-comet/pull/1540) (andygrove) - chore: Enable Comet explicitly in `CometTPCDSQueryTestSuite` [#1559](https://github.com/apache/datafusion-comet/pull/1559) (andygrove) - chore: Fix some inconsistencies in memory pool configuration [#1561](https://github.com/apache/datafusion-comet/pull/1561) (andygrove) - upgraded spark 3.5.4 to 3.5.5 [#1565](https://github.com/apache/datafusion-comet/pull/1565) (YanivKunda) - minor: fix typo [#1570](https://github.com/apache/datafusion-comet/pull/1570) (wForget) - Chore: simplify array related functions impl [#1490](https://github.com/apache/datafusion-comet/pull/1490) (kazantsev-maksim) - added fallback using reflection for backward-compatibility [#1573](https://github.com/apache/datafusion-comet/pull/1573) (YanivKunda) - chore: Override node name for CometSparkToColumnar [#1577](https://github.com/apache/datafusion-comet/pull/1577) (l0kr) - chore: Reimplement ShuffleWriterExec using interleave_record_batch [#1511](https://github.com/apache/datafusion-comet/pull/1511) (Kontinuation) - chore: Run Comet tests for more Spark versions [#1582](https://github.com/apache/datafusion-comet/pull/1582) (andygrove) - Feat: support array_except function [#1343](https://github.com/apache/datafusion-comet/pull/1343) (kazantsev-maksim) - minor: Fix clippy warnings [#1606](https://github.com/apache/datafusion-comet/pull/1606) (Kontinuation) - chore: Remove some unwraps in hashing code [#1600](https://github.com/apache/datafusion-comet/pull/1600) (andygrove) - chore: Remove redundant shims for getFailOnError [#1608](https://github.com/apache/datafusion-comet/pull/1608) (andygrove) - chore: Making comet native operators write spill files to spark local dir [#1581](https://github.com/apache/datafusion-comet/pull/1581) (Kontinuation) - chore: Refactor QueryPlanSerde to use idiomatic Scala and reduce verbosity [#1609](https://github.com/apache/datafusion-comet/pull/1609) (andygrove) - chore: Create simple fuzz test as part of test suite [#1610](https://github.com/apache/datafusion-comet/pull/1610) (andygrove) - chore: Document `testSingleLineQuery` test method [#1628](https://github.com/apache/datafusion-comet/pull/1628) (comphead) - chore: Parquet fuzz testing [#1623](https://github.com/apache/datafusion-comet/pull/1623) (andygrove) - chore: Change default Spark version to 3.5 [#1620](https://github.com/apache/datafusion-comet/pull/1620) (andygrove) - chore: Add manually-triggered CI jobs for testing Spark SQL with native scans [#1624](https://github.com/apache/datafusion-comet/pull/1624) (andygrove) - chore: refactor v2 scan conversion [#1621](https://github.com/apache/datafusion-comet/pull/1621) (andygrove) - chore: clean up `planner.rs` [#1650](https://github.com/apache/datafusion-comet/pull/1650) (comphead) - chore: correct name of pipelines for native_datafusion ci workflow [#1653](https://github.com/apache/datafusion-comet/pull/1653) (parthchandra) - chore: Upgrade to datafusion 47.0.0-rc1 and arrow-rs 55.0.0 [#1563](https://github.com/apache/datafusion-comet/pull/1563) (andygrove) - chore: Upgrade to datafusion 47.0.0 [#1663](https://github.com/apache/datafusion-comet/pull/1663) (YanivKunda) - chore: Enable CometFuzzTestSuite int96 test for experimental native scans (without complex types) [#1664](https://github.com/apache/datafusion-comet/pull/1664) (mbutrovich) - chore: Refactor Memory Pools [#1662](https://github.com/apache/datafusion-comet/pull/1662) (EmilyMatt) ## Credits Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. ``` 31 Andy Grove 11 Oleks V 10 Zhen Wang 7 Kristin Cowalcijk 6 Matt Butrovich 5 Parth Chandra 3 Emily Matheys 3 Yaniv Kunda 2 KAZUYUKI TANIMURA 2 Kazantsev Maksim 1 Łukasz ``` Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. ================================================ FILE: dev/changelog/0.9.0.md ================================================ # DataFusion Comet 0.9.0 Changelog This release consists of 139 commits from 24 contributors. See credits at the end of this changelog for more information. **Fixed bugs:** - fix: typo for `instr` in fuzz testing [#1686](https://github.com/apache/datafusion-comet/pull/1686) (mbutrovich) - fix: Bucketed scan fallback for native_datafusion Parquet scan [#1720](https://github.com/apache/datafusion-comet/pull/1720) (mbutrovich) - fix: Skip row index Spark SQL tests for native_datafusion Parquet scan [#1724](https://github.com/apache/datafusion-comet/pull/1724) (mbutrovich) - fix: Check acquired memory when CometMemoryPool grows [#1732](https://github.com/apache/datafusion-comet/pull/1732) (wForget) - fix: Fix data race in memory profiling [#1727](https://github.com/apache/datafusion-comet/pull/1727) (andygrove) - fix: Enable some DPP Spark SQL tests [#1734](https://github.com/apache/datafusion-comet/pull/1734) (andygrove) - fix: support literal null list and map [#1742](https://github.com/apache/datafusion-comet/pull/1742) (kazuyukitanimura) - fix: get_struct field is incorrect when struct in array [#1687](https://github.com/apache/datafusion-comet/pull/1687) (comphead) - fix: cast map types correctly in schema adapter [#1771](https://github.com/apache/datafusion-comet/pull/1771) (parthchandra) - fix: correct schema type checking in native_iceberg_compat [#1755](https://github.com/apache/datafusion-comet/pull/1755) (parthchandra) - fix: default values for native_datafusion scan [#1756](https://github.com/apache/datafusion-comet/pull/1756) (mbutrovich) - fix: [native_scans] Support `CASE_SENSITIVE` when reading Parquet [#1782](https://github.com/apache/datafusion-comet/pull/1782) (andygrove) - fix: cargo install tpchgen-cli in benchmark doc [#1797](https://github.com/apache/datafusion-comet/pull/1797) (zhuqi-lucas) - fix: support `map_keys` [#1788](https://github.com/apache/datafusion-comet/pull/1788) (comphead) - fix: fall back on nested types for default values [#1799](https://github.com/apache/datafusion-comet/pull/1799) (mbutrovich) - fix: Re-enable Spark 4 tests on Linux [#1806](https://github.com/apache/datafusion-comet/pull/1806) (andygrove) - fix: fallback to Spark scan if encryption is enabled (native_datafusion/native_iceberg_compat) [#1785](https://github.com/apache/datafusion-comet/pull/1785) (parthchandra) - fix: native_iceberg_compat: move checking parquet types above fetching batch [#1809](https://github.com/apache/datafusion-comet/pull/1809) (mbutrovich) - fix: translate missing or corrupt file exceptions, fall back if asked to ignore [#1765](https://github.com/apache/datafusion-comet/pull/1765) (mbutrovich) - fix: Fix Spark SQL AQE exchange reuse test failures [#1811](https://github.com/apache/datafusion-comet/pull/1811) (coderfender) - fix: Enable more Spark SQL tests [#1834](https://github.com/apache/datafusion-comet/pull/1834) (andygrove) - fix: support `map_values` [#1835](https://github.com/apache/datafusion-comet/pull/1835) (comphead) - fix: Handle case where num_cols == 0 in native execution [#1840](https://github.com/apache/datafusion-comet/pull/1840) (andygrove) - fix: Fix shuffle writing rows containing null struct fields [#1845](https://github.com/apache/datafusion-comet/pull/1845) (Kontinuation) - fix: Fall back to Spark for `RANGE BETWEEN` window expressions [#1848](https://github.com/apache/datafusion-comet/pull/1848) (andygrove) - fix: Remove COMET_SHUFFLE_FALLBACK_TO_COLUMNAR hack [#1865](https://github.com/apache/datafusion-comet/pull/1865) (andygrove) - fix: support read Struct by user schema [#1860](https://github.com/apache/datafusion-comet/pull/1860) (comphead) - fix: map parquet field_id correctly (native_iceberg_compat) [#1815](https://github.com/apache/datafusion-comet/pull/1815) (parthchandra) - fix: cast_struct_to_struct aligns to Spark behavior [#1879](https://github.com/apache/datafusion-comet/pull/1879) (mbutrovich) - fix: correctly handle schemas with nested array of struct (native_iceberg_compat) [#1883](https://github.com/apache/datafusion-comet/pull/1883) (parthchandra) - fix: set RangePartitioning for native shuffle default to false [#1907](https://github.com/apache/datafusion-comet/pull/1907) (mbutrovich) - fix: conflict between #1905 and #1892. [#1919](https://github.com/apache/datafusion-comet/pull/1919) (mbutrovich) - fix: Add overflow check to evaluate of sum decimal accumulator [#1922](https://github.com/apache/datafusion-comet/pull/1922) (leung-ming) - fix: Fix overflow handling when casting float to decimal [#1914](https://github.com/apache/datafusion-comet/pull/1914) (leung-ming) - fix: Ignore a test case fails on Miri [#1951](https://github.com/apache/datafusion-comet/pull/1951) (leung-ming) **Performance related:** - perf: Add memory profiling [#1702](https://github.com/apache/datafusion-comet/pull/1702) (andygrove) - perf: Add performance tracing capability [#1706](https://github.com/apache/datafusion-comet/pull/1706) (andygrove) - perf: Add `COMET_RESPECT_PARQUET_FILTER_PUSHDOWN` config [#1936](https://github.com/apache/datafusion-comet/pull/1936) (andygrove) **Implemented enhancements:** - feat: add jemalloc as optional custom allocator [#1679](https://github.com/apache/datafusion-comet/pull/1679) (mbutrovich) - feat: support `array_repeat` [#1680](https://github.com/apache/datafusion-comet/pull/1680) (comphead) - feat: More warning info for users [#1667](https://github.com/apache/datafusion-comet/pull/1667) (hsiang-c) - feat: decode() expression when using 'utf-8' encoding [#1697](https://github.com/apache/datafusion-comet/pull/1697) (mbutrovich) - feat: regexp_replace() expression with no starting offset [#1700](https://github.com/apache/datafusion-comet/pull/1700) (mbutrovich) - feat: Improve performance tracing feature [#1730](https://github.com/apache/datafusion-comet/pull/1730) (andygrove) - feat: Set/cancel with job tag and make max broadcast table size configurable [#1693](https://github.com/apache/datafusion-comet/pull/1693) (wForget) - feat: Add support for `expm1` expression from `datafusion-spark` crate [#1711](https://github.com/apache/datafusion-comet/pull/1711) (andygrove) - feat: Add config option for showing all Comet plan transformations [#1780](https://github.com/apache/datafusion-comet/pull/1780) (andygrove) - feat: Support Type widening: byte → short/int/long, short → int/long [#1770](https://github.com/apache/datafusion-comet/pull/1770) (huaxingao) - feat: Translate Hadoop S3A configurations to object_store configurations [#1817](https://github.com/apache/datafusion-comet/pull/1817) (Kontinuation) - feat: Upgrade to official DataFusion 48.0.0 release [#1877](https://github.com/apache/datafusion-comet/pull/1877) (andygrove) - feat: Add experimental auto mode for `COMET_PARQUET_SCAN_IMPL` [#1747](https://github.com/apache/datafusion-comet/pull/1747) (andygrove) - feat: support RangePartitioning with native shuffle [#1862](https://github.com/apache/datafusion-comet/pull/1862) (mbutrovich) - feat: Add support for signum expression [#1889](https://github.com/apache/datafusion-comet/pull/1889) (andygrove) - feat: Add support to lookup map by key [#1898](https://github.com/apache/datafusion-comet/pull/1898) (comphead) - feat: support array_max [#1892](https://github.com/apache/datafusion-comet/pull/1892) (drexler-sky) - feat: pass ignore_nulls flag to first and last [#1866](https://github.com/apache/datafusion-comet/pull/1866) (rluvaton) - feat: Implement ToPrettyString [#1921](https://github.com/apache/datafusion-comet/pull/1921) (andygrove) - feat: Support hadoop s3a config in native_iceberg_compat [#1925](https://github.com/apache/datafusion-comet/pull/1925) (parthchandra) - feat: rand expression support [#1199](https://github.com/apache/datafusion-comet/pull/1199) (akupchinskiy) - feat: supports array_distinct [#1923](https://github.com/apache/datafusion-comet/pull/1923) (drexler-sky) - feat: `auto` scan mode should check for supported file location [#1930](https://github.com/apache/datafusion-comet/pull/1930) (andygrove) - feat: Encapsulate Parquet objects [#1920](https://github.com/apache/datafusion-comet/pull/1920) (huaxingao) - feat: Change default value of `COMET_NATIVE_SCAN_IMPL` to `auto` [#1933](https://github.com/apache/datafusion-comet/pull/1933) (andygrove) - feat: Supports array_union [#1945](https://github.com/apache/datafusion-comet/pull/1945) (drexler-sky) **Documentation updates:** - docs: Add changelog for 0.8.0 [#1675](https://github.com/apache/datafusion-comet/pull/1675) (andygrove) - docs: Add instructions on running TPC-H on macOS [#1647](https://github.com/apache/datafusion-comet/pull/1647) (andygrove) - docs: Add documentation for accelerating Iceberg Parquet scans with Comet [#1683](https://github.com/apache/datafusion-comet/pull/1683) (andygrove) - docs: Add note on setting `core.abbrev` when generating diffs [#1735](https://github.com/apache/datafusion-comet/pull/1735) (andygrove) - docs: Remove outdated param in macos bench guide [#1748](https://github.com/apache/datafusion-comet/pull/1748) (ding-young) - docs: Add instructions for running individual Spark SQL tests from sbt [#1752](https://github.com/apache/datafusion-comet/pull/1752) (coderfender) - docs: Add documentation for native_datafusion Parquet scanner's S3 support [#1832](https://github.com/apache/datafusion-comet/pull/1832) (Kontinuation) - docs: Add docs stating that Comet does not support reading decimals encoded in Parquet BINARY format [#1895](https://github.com/apache/datafusion-comet/pull/1895) (andygrove) **Other:** - chore: Start 0.9.0 development [#1676](https://github.com/apache/datafusion-comet/pull/1676) (andygrove) - chore: Update viable crates [#1677](https://github.com/apache/datafusion-comet/pull/1677) (EmilyMatt) - chore: match Maven plugin versions with Spark 3.5 [#1668](https://github.com/apache/datafusion-comet/pull/1668) (hsiang-c) - chore: Remove fallback reason "because the children were not native" [#1672](https://github.com/apache/datafusion-comet/pull/1672) (andygrove) - chore: Rename `scalarExprToProto` to `scalarFunctionExprToProto` [#1688](https://github.com/apache/datafusion-comet/pull/1688) (comphead) - chore: fix build errors [#1690](https://github.com/apache/datafusion-comet/pull/1690) (comphead) - chore: Make Aggregate transformation more compact [#1670](https://github.com/apache/datafusion-comet/pull/1670) (EmilyMatt) - chore: update dev/release/rat_exclude_files.txt [#1689](https://github.com/apache/datafusion-comet/pull/1689) (hsiang-c) - chore: Move Comet rules into their own files [#1695](https://github.com/apache/datafusion-comet/pull/1695) (andygrove) - chore: Remove fast encoding option [#1703](https://github.com/apache/datafusion-comet/pull/1703) (andygrove) - chore: fix CI job name [#1712](https://github.com/apache/datafusion-comet/pull/1712) (hsiang-c) - minor: Warn if memory pool is dropped with bytes still reserved [#1721](https://github.com/apache/datafusion-comet/pull/1721) (andygrove) - chore: Correct memory acquired size in unified memory pool [#1738](https://github.com/apache/datafusion-comet/pull/1738) (zuston) - chore: allow large errors for Clippy [#1743](https://github.com/apache/datafusion-comet/pull/1743) (comphead) - chore: Refactor DataTypeSupport [#1741](https://github.com/apache/datafusion-comet/pull/1741) (andygrove) - chore: More refactoring of type checking logic [#1744](https://github.com/apache/datafusion-comet/pull/1744) (andygrove) - chore: Enable more complex type tests [#1753](https://github.com/apache/datafusion-comet/pull/1753) (andygrove) - chore: Add `scanImpl` attribute to `CometScanExec` [#1746](https://github.com/apache/datafusion-comet/pull/1746) (andygrove) - chore: Prepare for DataFusion 48.0.0 [#1710](https://github.com/apache/datafusion-comet/pull/1710) (andygrove) - Docs: Setup Comet on IntelliJ [#1760](https://github.com/apache/datafusion-comet/pull/1760) (coderfender) - chore: Reenable nested types for CometFuzzTestSuite with int96 [#1761](https://github.com/apache/datafusion-comet/pull/1761) (mbutrovich) - chore: Enable partial Spark SQL tests for `native_iceberg_compat` scan [#1762](https://github.com/apache/datafusion-comet/pull/1762) (andygrove) - chore: [native_iceberg_compat / native_datafusion] Ignore Spark SQL Parquet encryption tests [#1763](https://github.com/apache/datafusion-comet/pull/1763) (andygrove) - build: Ignore array_repeat test to fix CI issues [#1774](https://github.com/apache/datafusion-comet/pull/1774) (andygrove) - chore: Upload crash logs if Java tests fail [#1779](https://github.com/apache/datafusion-comet/pull/1779) (andygrove) - chore: Drop support for Java 8 [#1777](https://github.com/apache/datafusion-comet/pull/1777) (andygrove) - chore: Bump arrow to 18.3.0 [#1773](https://github.com/apache/datafusion-comet/pull/1773) (Kontinuation) - build: Stop running Comet's Spark 4 tests on Linux for PR builds [#1802](https://github.com/apache/datafusion-comet/pull/1802) (andygrove) - Chore: Moved strings expressions to separate file [#1792](https://github.com/apache/datafusion-comet/pull/1792) (kazantsev-maksim) - chore: Speed up "PR Builds" CI workflows [#1807](https://github.com/apache/datafusion-comet/pull/1807) (andygrove) - chore: [native scans] Ignore Spark SQL test for string predicate pushdown [#1768](https://github.com/apache/datafusion-comet/pull/1768) (andygrove) - chore: Bump DataFusion to git rev 2c2f225 [#1814](https://github.com/apache/datafusion-comet/pull/1814) (andygrove) - Feat: support bit_count function [#1602](https://github.com/apache/datafusion-comet/pull/1602) (kazantsev-maksim) - Chore: implement bit_not as ScalarUDFImpl [#1825](https://github.com/apache/datafusion-comet/pull/1825) (kazantsev-maksim) - build: Specify -Dsbt.log.noformat=true in sbt CI runs [#1822](https://github.com/apache/datafusion-comet/pull/1822) (andygrove) - chore: Use unique artifact names in Java test run [#1818](https://github.com/apache/datafusion-comet/pull/1818) (andygrove) - minor: Refactor PhysicalPlanner::default() to avoid duplicate code [#1821](https://github.com/apache/datafusion-comet/pull/1821) (andygrove) - Chore: implement bit_count as ScalarUDFImpl [#1826](https://github.com/apache/datafusion-comet/pull/1826) (kazantsev-maksim) - chore: IgnoreCometNativeScan on a few more Spark SQL tests [#1837](https://github.com/apache/datafusion-comet/pull/1837) (mbutrovich) - chore: Enable tests in RemoveRedundantProjectsSuite.scala related to issue #242 [#1838](https://github.com/apache/datafusion-comet/pull/1838) (rishvin) - minor: Replace many instances of `checkSparkAnswer` with `checkSparkAnswerAndOperator` [#1851](https://github.com/apache/datafusion-comet/pull/1851) (andygrove) - chore: Update documentation and ignore Spark SQL tests for known issue with count distinct on NaN in aggregate [#1847](https://github.com/apache/datafusion-comet/pull/1847) (andygrove) - chore: Ignore Spark SQL WholeStageCodegenSuite tests [#1859](https://github.com/apache/datafusion-comet/pull/1859) (andygrove) - chore: Upgrade to DataFusion 48.0.0-rc3 [#1863](https://github.com/apache/datafusion-comet/pull/1863) (andygrove) - upgraded spark 3.5.5 to 3.5.6 [#1861](https://github.com/apache/datafusion-comet/pull/1861) (YanivKunda) - build: Disable some rounding tests when miri is enabled [#1873](https://github.com/apache/datafusion-comet/pull/1873) (andygrove) - chore: Enable Spark SQL tests for `native_iceberg_compat` [#1876](https://github.com/apache/datafusion-comet/pull/1876) (andygrove) - chore: Enable more Spark SQL tests [#1869](https://github.com/apache/datafusion-comet/pull/1869) (andygrove) - chore: refactor planner read schema tests [#1886](https://github.com/apache/datafusion-comet/pull/1886) (comphead) - chore: Implement date_trunc as ScalarUDFImpl [#1880](https://github.com/apache/datafusion-comet/pull/1880) (leung-ming) - Chore: implement datetime funcs as ScalarUDFImpl [#1874](https://github.com/apache/datafusion-comet/pull/1874) (trompa) - minor: Improve testing of math scalar functions [#1896](https://github.com/apache/datafusion-comet/pull/1896) (andygrove) - minor: Avoid rewriting join to unsupported join [#1888](https://github.com/apache/datafusion-comet/pull/1888) (andygrove) - chore: Enable `native_iceberg_compat` Spark SQL tests (for real, this time) [#1910](https://github.com/apache/datafusion-comet/pull/1910) (andygrove) - chore: rename makeParquetFileAllTypes to makeParquetFileAllPrimitiveTypes [#1905](https://github.com/apache/datafusion-comet/pull/1905) (parthchandra) - chore: add a test case to read from an arbitrarily complex type schema [#1911](https://github.com/apache/datafusion-comet/pull/1911) (parthchandra) - test: Trigger Spark 3.4.3 SQL tests for iceberg-compat [#1912](https://github.com/apache/datafusion-comet/pull/1912) (kazuyukitanimura) - build: Fix conflict between #1910 and #1912 [#1924](https://github.com/apache/datafusion-comet/pull/1924) (andygrove) - minor: fix kube/Dockerfile build failed [#1918](https://github.com/apache/datafusion-comet/pull/1918) (zhangxffff) - chore: Improve reporting of fallback reasons for CollectLimit [#1694](https://github.com/apache/datafusion-comet/pull/1694) (andygrove) - chore: move udf registration to better place [#1899](https://github.com/apache/datafusion-comet/pull/1899) (rluvaton) - chore: Comet + Iceberg (1.8.1) CI [#1715](https://github.com/apache/datafusion-comet/pull/1715) (hsiang-c) - chore: Introduce `exprHandlers` map in QueryPlanSerde [#1903](https://github.com/apache/datafusion-comet/pull/1903) (andygrove) - chore: Enable Spark SQL tests for auto scan mode [#1885](https://github.com/apache/datafusion-comet/pull/1885) (andygrove) - Feat: support bit_get function [#1713](https://github.com/apache/datafusion-comet/pull/1713) (kazantsev-maksim) - chore: Clippy fixes for Rust 1.88 [#1939](https://github.com/apache/datafusion-comet/pull/1939) (andygrove) - Minor: Add unit tests for `ceil`/`floor` functions [#1728](https://github.com/apache/datafusion-comet/pull/1728) (tlm365) ## Credits Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. ``` 62 Andy Grove 16 Matt Butrovich 10 Oleks V 8 Parth Chandra 5 Kazantsev Maksim 5 hsiang-c 4 Kristin Cowalcijk 4 Leung Ming 3 B Vadlamani 3 drexler-sky 2 Emily Matheys 2 Huaxin Gao 2 KAZUYUKI TANIMURA 2 Raz Luvaton 2 Zhen Wang 1 Artem Kupchinskiy 1 Junfan Zhang 1 Qi Zhu 1 Rishab Joshi 1 Tai Le Manh 1 Yaniv Kunda 1 Zhang Xiaofeng 1 ding-young 1 trompa ``` Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. ================================================ FILE: dev/changelog/0.9.1.md ================================================ # DataFusion Comet 0.9.1 Changelog This release consists of 2 commits from 1 contributors. See credits at the end of this changelog for more information. **Fixed bugs:** - fix: [branch-0.9] Backport FFI fix [#2164](https://github.com/apache/datafusion-comet/pull/2164) (andygrove) - fix: [branch-0.9] Avoid double free in CometUnifiedShuffleMemoryAllocator [#2201](https://github.com/apache/datafusion-comet/pull/2201) (andygrove) ## Credits Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. ``` 2 Andy Grove ``` Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. ================================================ FILE: dev/checkstyle-suppressions.xml ================================================ ================================================ FILE: dev/ci/check-suites.py ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. import sys from pathlib import Path def file_to_class_name(path: Path) -> str | None: parts = path.parts if "org" not in parts or "apache" not in parts: return None org_index = parts.index("org") package_parts = parts[org_index:] class_name = ".".join(package_parts) class_name = class_name.replace(".scala", "") return class_name if __name__ == "__main__": # ignore traits, abstract classes, and intentionally skipped test suites ignore_list = [ "org.apache.comet.parquet.ParquetReadSuite", # abstract "org.apache.comet.parquet.ParquetReadFromS3Suite", # manual test suite "org.apache.comet.IcebergReadFromS3Suite", # manual test suite "org.apache.spark.sql.comet.CometPlanStabilitySuite", # abstract "org.apache.spark.sql.comet.ParquetDatetimeRebaseSuite", # abstract "org.apache.comet.exec.CometColumnarShuffleSuite" # abstract ] for workflow_filename in [".github/workflows/pr_build_linux.yml", ".github/workflows/pr_build_macos.yml"]: workflow = open(workflow_filename, encoding="utf-8").read() root = Path(".") for path in root.rglob("*Suite.scala"): class_name = file_to_class_name(path) if class_name: if "Shim" in class_name: continue if class_name in ignore_list: continue if class_name not in workflow: print(f"Suite not found in workflow {workflow_filename}: {class_name}") sys.exit(-1) print(f"Found {class_name} in {workflow_filename}") ================================================ FILE: dev/ci/check-working-tree-clean.sh ================================================ #!/bin/bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # set -euo pipefail # Exit on errors, undefined vars, pipe failures # Check if we're in a git repository if ! git rev-parse --git-dir > /dev/null 2>&1; then echo "Error: Not in a git repository" # exit 1 fi # Fail if there are any local changes (staged, unstaged, or untracked) if [ -n "$(git status --porcelain)" ]; then echo "Working tree is not clean:" git status --short echo "Full diff:" git diff echo "" echo "Please commit, stash, or clean these changes before proceeding." exit 1 else echo "Working tree is clean" fi ================================================ FILE: dev/copyright/java-header.txt ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ ================================================ FILE: dev/diffs/3.4.3.diff ================================================ diff --git a/pom.xml b/pom.xml index d3544881af1..377683b10c5 100644 --- a/pom.xml +++ b/pom.xml @@ -148,6 +148,8 @@ 0.10.0 2.5.1 2.0.8 + 3.4 + 0.15.0-SNAPSHOT 2.5.1 2.0.8 + 3.5 + 0.15.0-SNAPSHOT org.apache.datasketches diff --git a/sql/core/pom.xml b/sql/core/pom.xml index dcf6223a98b..0458a5bb640 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -90,6 +90,10 @@ org.apache.spark spark-tags_${scala.binary.version} + + org.apache.datafusion + comet-spark-spark${spark.version.short}_${scala.binary.version} + \n""") print(f"# DataFusion Comet {version} Changelog\n") # get the number of commits commit_count = subprocess.check_output(f"git log --pretty=oneline {tag1}..{tag2} | wc -l", shell=True, text=True).strip() # get number of contributors contributor_count = subprocess.check_output(f"git shortlog -sn {tag1}..{tag2} | wc -l", shell=True, text=True).strip() print(f"This release consists of {commit_count} commits from {contributor_count} contributors. " f"See credits at the end of this changelog for more information.\n") print_pulls(repo_name, "Breaking changes", breaking) print_pulls(repo_name, "Fixed bugs", bugs) print_pulls(repo_name, "Performance related", performance) print_pulls(repo_name, "Implemented enhancements", enhancements) print_pulls(repo_name, "Documentation updates", docs) print_pulls(repo_name, "Other", other) # show code contributions credits = subprocess.check_output(f"git shortlog -sn {tag1}..{tag2}", shell=True, text=True).rstrip() print("## Credits\n") print("Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) " "per contributor.\n") print("```") print(credits) print("```\n") print("Thank you also to everyone who contributed in other ways such as filing issues, reviewing " "PRs, and providing feedback on this release.\n") def resolve_ref(ref): """Resolve a git ref (e.g. HEAD, branch name) to a full commit SHA.""" try: return subprocess.check_output( ["git", "rev-parse", ref], text=True ).strip() except subprocess.CalledProcessError: # If it can't be resolved locally, return as-is (e.g. a tag name # that the GitHub API can resolve) return ref def cli(args=None): """Process command line arguments.""" if not args: args = sys.argv[1:] parser = argparse.ArgumentParser() parser.add_argument("tag1", help="The previous commit or tag (e.g. 0.1.0)") parser.add_argument("tag2", help="The current commit or tag (e.g. HEAD)") parser.add_argument("version", help="The version number to include in the changelog") args = parser.parse_args() # Resolve refs to SHAs so the GitHub API compares the same commits # as the local git log. Without this, refs like HEAD get resolved by # the GitHub API to the default branch instead of the current branch. tag1 = resolve_ref(args.tag1) tag2 = resolve_ref(args.tag2) token = os.getenv("GITHUB_TOKEN") project = "apache/datafusion-comet" g = Github(token) repo = g.get_repo(project) generate_changelog(repo, project, tag1, tag2, args.version) if __name__ == "__main__": cli() ================================================ FILE: dev/release/publish-to-maven.sh ================================================ #!/bin/bash # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ### # This file is based on the script release-build.sh in Spark ### function usage { local NAME=$(basename $0) cat << EOF usage: $NAME options Publish signed artifacts to Maven. Options -u ASF_USERNAME - Username of ASF committer account -r LOCAL_REPO - path to temporary local maven repo (created and written to by 'build-release-comet.sh') The following will be prompted for - ASF_PASSWORD - Password of ASF committer account GPG_KEY - GPG key used to sign release artifacts GPG_PASSPHRASE - Passphrase for GPG key EOF exit 1 } SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" COMET_HOME_DIR=$SCRIPT_DIR/../.. ASF_USERNAME="" ASF_PASSWORD="" NEXUS_ROOT=https://repository.apache.org/service/local/staging NEXUS_PROFILE=789e15c00fd47 while getopts "u:r:h" opt; do case $opt in u) ASF_USERNAME="$OPTARG" ;; r) LOCAL_REPO="$OPTARG" ;; h) usage ;; \?) error "Invalid option. Run with -h for help." ;; esac done if [ "$LOCAL_REPO" == "" ] then "Please provide the local Maven repository (from 'build-release-comet.sh')" usage fi if [ "$ASF_USERNAME" == "" ] then read -p "ASF Username : " ASF_USERNAME && echo "" fi # Read some secret information read -s -p "ASF Password : " ASF_PASSWORD && echo "" read -s -p "GPG Key (Optional): " GPG_KEY && echo "" read -s -p "GPG Passphrase : " GPG_PASSPHRASE && echo "" if [ "$ASF_USERNAME" == "" ] || [ "$ASF_PASSWORD" == "" ] || [ "$GPG_PASSPHRASE" = "" ] then echo "Missing credentials" exit 1 fi # Default GPG command to use GPG="gpg --pinentry-mode loopback" if [ "$GPG_KEY" != "" ] then GPG="$GPG -u $GPG_KEY" fi # sha1sum on linux, shasum on macos SHA1SUM=$( which sha1sum || which shasum) GIT_HASH=$(git rev-parse --short HEAD) # REF: https://support.sonatype.com/hc/en-us/articles/213465818-How-can-I-programmatically-upload-an-artifact-into-Nexus-Repo-2 # REF: https://support.sonatype.com/hc/en-us/articles/213465868-Uploading-to-a-Nexus-Repository-2-Staging-Repository-via-REST-API echo "Creating Nexus staging repository" REPO_REQUEST="Apache Datafusion Comet $COMET_VERSION (commit $GIT_HASH)" REPO_REQUEST_RESPONSE=$(curl -X POST -d "$REPO_REQUEST" -u $ASF_USERNAME:$ASF_PASSWORD \ -H "Content-Type:application/xml" \ $NEXUS_ROOT/profiles/$NEXUS_PROFILE/start) if [ $? -ne 0 ] then echo "Error creating staged repository" echo "$REPO_REQUEST_RESPONSE" exit 1 fi STAGED_REPO_ID=$(echo $REPO_REQUEST_RESPONSE | xmllint --xpath "//stagedRepositoryId/text()" -) echo "Created Nexus staging repository: $STAGED_REPO_ID" if [ "$STAGED_REPO_ID" == "" ] then echo "Error creating staged repository" echo "$REPO_REQUEST_RESPONSE" exit 1 fi echo "Deploying artifacts from $LOCAL_REPO" pushd $LOCAL_REPO/org/apache/datafusion # Remove any extra files generated during install find . -type f |grep -v \.jar |grep -v \.pom | xargs rm echo "Creating hash and signature files" for file in $(find . -type f) do echo $GPG_PASSPHRASE | $GPG --passphrase-fd 0 --output $file.asc \ --detach-sig --armour $file; if [ $(command -v md5) ]; then # Available on macOS; -q to keep only hash md5 -q $file > $file.md5 else # Available on Linux; cut to keep only hash md5sum $file | cut -f1 -d' ' > $file.md5 fi $SHA1SUM $file | cut -f1 -d' ' > $file.sha1 done NEXUS_UPLOAD=$NEXUS_ROOT/deployByRepositoryId/$STAGED_REPO_ID echo "Uploading files to $NEXUS_UPLOAD" for file in $(find . -type f) do # strip leading ./ FILE_SHORT=$(echo $file | sed -e "s/\.\///") DEST_URL="$NEXUS_UPLOAD/org/apache/datafusion/$FILE_SHORT" echo " Uploading $FILE_SHORT" curl -u $ASF_USERNAME:$ASF_PASSWORD --upload-file $FILE_SHORT $DEST_URL if [ $? -ne 0 ] then echo " - Failed" exit 2 fi done echo "Closing nexus staging repository" REPO_REQUEST="$STAGED_REPO_IDApache Datafusion Comet $COMET_VERSION (commit $GIT_HASH)" REPO_REQUEST_RESPONSE=$(curl -X POST -d "$REPO_REQUEST" -u $ASF_USERNAME:$ASF_PASSWORD \ -H "Content-Type:application/xml" -v \ $NEXUS_ROOT/profiles/$NEXUS_PROFILE/finish) echo "Closed Nexus staging repository: $STAGED_REPO_ID" popd ================================================ FILE: dev/release/rat_exclude_files.txt ================================================ *.gitignore *.dockerignore .github/pull_request_template.md .gitmodules native/Cargo.lock native/testdata/backtrace.txt native/testdata/stacktrace.txt native/core/testdata/backtrace.txt native/core/testdata/stacktrace.txt dev/copyright/scala-header.txt dev/diffs/iceberg/*.diff dev/release/requirements.txt dev/release/rat_exclude_files.txt docs/comet-* docs/source/_static/images/*.svg docs/source/_static/images/comet-dataflow.excalidraw docs/source/contributor-guide/benchmark-results/**/*.json docs/logos/*.png docs/logos/*.svg docs/source/contributor-guide/*.svg rust-toolchain spark/src/test/resources/tpcds-extended/q*.sql spark/src/test/resources/tpcds-query-results/*.out spark/src/test/resources/tpcds-micro-benchmarks/*.sql spark/src/test/resources/tpcds-plan-stability/approved-plans*/**/explain.txt spark/src/test/resources/tpcds-plan-stability/approved-plans*/**/simplified.txt spark/src/test/resources/tpcds-plan-stability/approved-plans*/**/extended.txt spark/src/test/resources/tpch-query-results/*.out spark/src/test/resources/tpch-extended/q*.sql spark/src/test/resources/test-data/*.csv spark/src/test/resources/test-data/*.ndjson spark/inspections/CometTPC*results.txt benchmarks/tpc/queries/**/*.sql ================================================ FILE: dev/release/release-tarball.sh ================================================ #!/bin/bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # # Adapted from https://github.com/apache/arrow-rs/tree/master/dev/release/release-tarball.sh # This script copies a tarball from the "dev" area of the # dist.apache.datafusion repository to the "release" area # # This script should only be run after the release has been approved # by the Apache DataFusion PMC committee. # # See release/README.md for full release instructions # # Based in part on post-01-upload.sh from apache/arrow set -e set -u if [ "$#" -ne 2 ]; then echo "Usage: $0 " echo "ex. $0 4.1.0 2" exit fi version=$1 rc=$2 read -r -p "Proceed to release tarball for ${version}-rc${rc}? [y/N]: " answer answer=${answer:-no} if [ "${answer}" != "y" ]; then echo "Cancelled tarball release!" exit 1 fi tmp_dir=tmp-apache-datafusion-comet-dist echo "Recreate temporary directory: ${tmp_dir}" rm -rf ${tmp_dir} mkdir -p ${tmp_dir} echo "Clone dev dist repository" svn \ co \ https://dist.apache.org/repos/dist/dev/datafusion/apache-datafusion-comet-${version}-rc${rc} \ ${tmp_dir}/dev echo "Clone release dist repository" svn co https://dist.apache.org/repos/dist/release/datafusion ${tmp_dir}/release echo "Copy ${version}-rc${rc} to release working copy" release_version=datafusion-comet-${version} mkdir -p ${tmp_dir}/release/${release_version} cp -r ${tmp_dir}/dev/* ${tmp_dir}/release/${release_version}/ svn add ${tmp_dir}/release/${release_version} echo "Commit release" svn ci -m "Apache DataFusion Comet ${version}" ${tmp_dir}/release echo "Clean up" rm -rf ${tmp_dir} echo "Success! The release is available here:" echo " https://dist.apache.org/repos/dist/release/datafusion/${release_version}" ================================================ FILE: dev/release/requirements.txt ================================================ PyGitHub ================================================ FILE: dev/release/run-rat.sh ================================================ #!/bin/bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # RAT_VERSION=0.16.1 # download apache rat if [ ! -f apache-rat-${RAT_VERSION}.jar ]; then curl -s https://repo1.maven.org/maven2/org/apache/rat/apache-rat/${RAT_VERSION}/apache-rat-${RAT_VERSION}.jar > apache-rat-${RAT_VERSION}.jar fi RAT="java -jar apache-rat-${RAT_VERSION}.jar -x " RELEASE_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd) # generate the rat report $RAT $1 > rat.txt python3 $RELEASE_DIR/check-rat-report.py $RELEASE_DIR/rat_exclude_files.txt rat.txt > filtered_rat.txt UNAPPROVED=`cat filtered_rat.txt | grep "NOT APPROVED" | wc -l` if [ "0" -eq "${UNAPPROVED}" ]; then echo "No unapproved licenses" else echo "${UNAPPROVED} unapproved licences. Check rat report: rat.txt" cat filtered_rat.txt exit 1 fi ================================================ FILE: dev/release/verify-release-candidate.sh ================================================ #!/bin/bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # case $# in 2) VERSION="$1" RC_NUMBER="$2" ;; *) echo "Usage: $0 X.Y.Z RC_NUMBER" exit 1 ;; esac set -e set -x set -o pipefail SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" COMET_DIR="$(dirname $(dirname ${SOURCE_DIR}))" COMET_DIST_URL='https://dist.apache.org/repos/dist/dev/datafusion' download_dist_file() { curl \ --silent \ --show-error \ --fail \ --location \ --remote-name $COMET_DIST_URL/$1 } download_rc_file() { download_dist_file apache-datafusion-comet-${VERSION}-rc${RC_NUMBER}/$1 } import_gpg_keys() { download_dist_file KEYS gpg --import KEYS } if type shasum >/dev/null 2>&1; then sha256_verify="shasum -a 256 -c" sha512_verify="shasum -a 512 -c" else sha256_verify="sha256sum -c" sha512_verify="sha512sum -c" fi fetch_archive() { local dist_name=$1 download_rc_file ${dist_name}.tar.gz download_rc_file ${dist_name}.tar.gz.asc download_rc_file ${dist_name}.tar.gz.sha256 download_rc_file ${dist_name}.tar.gz.sha512 verify_dir_artifact_signatures } verify_dir_artifact_signatures() { # verify the signature and the checksums of each artifact find . -name '*.asc' | while read sigfile; do artifact=${sigfile/.asc/} gpg --verify $sigfile $artifact || exit 1 # go into the directory because the checksum files contain only the # basename of the artifact pushd $(dirname $artifact) base_artifact=$(basename $artifact) ${sha256_verify} $base_artifact.sha256 || exit 1 ${sha512_verify} $base_artifact.sha512 || exit 1 popd done } setup_tempdir() { cleanup() { if [ "${TEST_SUCCESS}" = "yes" ]; then rm -fr "${COMET_TMPDIR}" else echo "Failed to verify release candidate. See ${COMET_TMPDIR} for details." fi } if [ -z "${COMET_TMPDIR}" ]; then # clean up automatically if COMET_TMPDIR is not defined COMET_TMPDIR=$(mktemp -d -t "$1.XXXXX") trap cleanup EXIT else # don't clean up automatically mkdir -p "${COMET_TMPDIR}" fi } test_source_distribution() { set -e pushd native RUSTFLAGS="-Ctarget-cpu=native" cargo build --release popd # test with the latest supported version of Spark ./mvnw verify -Prelease -DskipTests -P"spark-3.4" -Dmaven.gitcommitid.skip=true } TEST_SUCCESS=no setup_tempdir "datafusion-comet-${VERSION}" echo "Working in sandbox ${COMET_TMPDIR}" cd ${COMET_TMPDIR} dist_name="apache-datafusion-comet-${VERSION}" import_gpg_keys fetch_archive ${dist_name} tar xf ${dist_name}.tar.gz pushd ${dist_name} test_source_distribution popd TEST_SUCCESS=yes echo 'Release candidate looks good!' exit 0 ================================================ FILE: dev/release/verifying-release-candidates.md ================================================ # Verifying DataFusion Comet Release Candidates The `dev/release/verify-release-candidate.sh` script in this repository can assist in the verification process. It checks the hashes and runs the build. It does not run the test suite because this takes a long time for this project and the test suites already run in CI before we create the release candidate, so running them again is somewhat redundant. ```shell ./dev/release/verify-release-candidate.sh 0.1.0 1 ``` The following command can be used to build a release for testing. ```shell make release-nogit ``` We hope that users will verify the release beyond running this script by testing the release candidate with their existing Spark jobs and report any functional issues or performance regressions. The email announcing the vote should contain a link to pre-built jar files in a Maven staging repository. Another way of verifying the release is to follow the [Comet Benchmarking Guide](https://datafusion.apache.org/comet/contributor-guide/benchmarking.html) and compare performance with the previous release. ================================================ FILE: dev/scalastyle-config.xml ================================================ Scalastyle standard configuration true ARROW, EQUALS, ELSE, TRY, CATCH, FINALLY, LARROW, RARROW ARROW, EQUALS, COMMA, COLON, IF, ELSE, DO, WHILE, FOR, MATCH, TRY, CATCH, FINALLY, LARROW, RARROW ^println$ @VisibleForTesting Runtime\.getRuntime\.addShutdownHook mutable\.SynchronizedBuffer Class\.forName Await\.result Await\.ready (\.toUpperCase|\.toLowerCase)(?!(\(|\(Locale.ROOT\))) throw new \w+Error\( org\.apache\.commons\.lang\. Use Commons Lang 3 classes (package org.apache.commons.lang3.*) instead of Commons Lang 2 (package org.apache.commons.lang.*) FileSystem.get\([a-zA-Z_$][a-zA-Z_$0-9]*\) java,scala,org,apache,3rdParty,comet javax?\..* scala\..* org\.(?!apache).* org\.apache\.(?!comet).* (?!(javax?\.|scala\.|org\.apache\.comet\.)).* org\.apache\.comet\..* COMMA \)\{ (?m)^(\s*)/[*][*].*$(\r|)\n^\1 [*] Use Javadoc style indentation for multiline comments case[^\n>]*=>\s*\{ Omit braces in case clauses. new (java\.lang\.)?(Byte|Integer|Long|Short)\( Use static factory 'valueOf' or 'parseXXX' instead of the deprecated constructors. Please use Apache Log4j 2 instead. 800> 30 10 50 -1,0,1,2,3 Objects.toStringHelper Avoid using Object.toStringHelper. Use ToStringBuilder instead. ================================================ FILE: docs/.gitignore ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. build temp venv/ .python-version comet-* ================================================ FILE: docs/Makefile ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # # Minimal makefile for Sphinx documentation # # You can set these variables from the command line, and also # from the environment for the first two. SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build SOURCEDIR = source BUILDDIR = build # Put it first so that "make" without argument is like "make help". help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) .PHONY: help Makefile # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) ================================================ FILE: docs/README.md ================================================ # Apache DataFusion Comet Documentation This folder contains the source content for the Apache DataFusion Comet documentation site. This content is published to when any changes are merged into the main branch. ## Dependencies It's recommended to install build dependencies and build the documentation inside a Python virtualenv. - Python - `pip install -r requirements.txt` ## Build & Preview Run the provided script to build the HTML pages. ```bash ./build.sh ``` The HTML will be generated into a `build` directory. Preview the site on Linux by running this command. ```bash firefox build/html/index.html ``` ## Making Changes To make changes to the docs, simply make a Pull Request with your proposed changes as normal. When the PR is merged the docs will be automatically updated. ## Release Process This documentation is hosted at When the PR is merged to the `main` branch of the `datafusion-comet` repository, a [GitHub workflow](https://github.com/apache/datafusion-comet/blob/main/.github/workflows/docs.yaml) which: 1. Builds the html content 2. Pushes the html content to the [`asf-site`](https://github.com/apache/datafusion-comet/tree/asf-site) branch in this repository. The Apache Software Foundation provides , which serves content based on the configuration in [.asf.yaml](https://github.com/apache/datafusion-comet/blob/main/.asf.yaml), which specifies the target as . ================================================ FILE: docs/build.sh ================================================ #!/bin/bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # set -e rm -rf build 2> /dev/null rm -rf temp 2> /dev/null mkdir temp cp -rf source/* temp/ # Add user guide from published releases rm -rf comet-0.11 rm -rf comet-0.12 rm -rf comet-0.13 python3 generate-versions.py # Generate dynamic content (configs, compatibility matrices) for latest docs # This runs GenerateDocs against the temp copy, not source files echo "Generating dynamic documentation content..." cd .. ./mvnw -q package -Pgenerate-docs -DskipTests -Dmaven.test.skip=true \ -Dexec.arguments="$(pwd)/docs/temp/user-guide/latest/" cd docs make SOURCEDIR=`pwd`/temp html ================================================ FILE: docs/generate-versions.py ================================================ #!/usr/bin/python ############################################################################## # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. ############################################################################## # This script clones release branches such as branch-0.9 and then copies the user guide markdown # for inclusion in the published documentation so that we publish user guides for released versions # of Comet import os import xml.etree.ElementTree as ET from pathlib import Path def get_major_minor_version(version: str): parts = version.split('.') return f"{parts[0]}.{parts[1]}" def get_version_from_pom(): pom_path = Path(__file__).parent.parent / "pom.xml" tree = ET.parse(pom_path) root = tree.getroot() namespace = {'maven': 'http://maven.apache.org/POM/4.0.0'} version_element = root.find('maven:version', namespace) if version_element is not None: return version_element.text else: raise ValueError("Could not find version in pom.xml") def replace_in_files(root: str, filename_pattern: str, search: str, replace: str): root_path = Path(root) for file in root_path.rglob(filename_pattern): text = file.read_text(encoding="utf-8") updated = text.replace(search, replace) if text != updated: file.write_text(updated, encoding="utf-8") print(f"Replaced {search} with {replace} in {file}") def insert_warning_after_asf_header(root: str, warning: str): root_path = Path(root) for file in root_path.rglob("*.md"): lines = file.read_text(encoding="utf-8").splitlines(keepends=True) new_lines = [] inserted = False for line in lines: new_lines.append(line) if not inserted and "-->" in line: new_lines.append(warning + "\n") inserted = True file.write_text("".join(new_lines), encoding="utf-8") def get_user_guide_dir(major_minor: str): if major_minor == "0.8" or major_minor == "0.9": return "docs/source/user-guide" else: return "docs/source/user-guide/latest" def publish_released_version(version: str): major_minor = get_major_minor_version(version) dir = get_user_guide_dir(major_minor) os.system(f"git clone --depth 1 https://github.com/apache/datafusion-comet.git -b branch-{major_minor} comet-{major_minor}") os.system(f"mkdir temp/user-guide/{major_minor}") os.system(f"cp -rf comet-{major_minor}/{dir}/* temp/user-guide/{major_minor}") # Replace $COMET_VERSION with actual version for file_pattern in ["*.md", "*.rst"]: replace_in_files(f"temp/user-guide/{major_minor}", file_pattern, "$COMET_VERSION", version) def generate_docs(snapshot_version: str, latest_released_version: str, previous_versions: list[str]): # Replace $COMET_VERSION with actual version for snapshot version for file_pattern in ["*.md", "*.rst"]: replace_in_files(f"temp/user-guide/latest", file_pattern, "$COMET_VERSION", snapshot_version) # Add user guide content for latest released versions publish_released_version(latest_released_version) # Add user guide content for older released versions for version in previous_versions: publish_released_version(version) # add warning that this is out-of-date documentation warning = f"""```{{warning}} This is **out-of-date** documentation. The latest Comet release is version {latest_released_version}. ```""" major_minor = get_major_minor_version(version) insert_warning_after_asf_header(f"temp/user-guide/{major_minor}", warning) if __name__ == "__main__": print("Generating versioned user guide docs...") snapshot_version = get_version_from_pom() latest_released_version = "0.14.0" previous_versions = ["0.11.0", "0.12.0", "0.13.0"] generate_docs(snapshot_version, latest_released_version, previous_versions) ================================================ FILE: docs/make.bat ================================================ @rem Licensed to the Apache Software Foundation (ASF) under one @rem or more contributor license agreements. See the NOTICE file @rem distributed with this work for additional information @rem regarding copyright ownership. The ASF licenses this file @rem to you under the Apache License, Version 2.0 (the @rem "License"); you may not use this file except in compliance @rem with the License. You may obtain a copy of the License at @rem @rem http://www.apache.org/licenses/LICENSE-2.0 @rem @rem Unless required by applicable law or agreed to in writing, @rem software distributed under the License is distributed on an @rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @rem KIND, either express or implied. See the License for the @rem specific language governing permissions and limitations @rem under the License. @ECHO OFF pushd %~dp0 REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) set SOURCEDIR=source set BUILDDIR=build if "%1" == "" goto help %SPHINXBUILD% >NUL 2>NUL if errorlevel 9009 ( echo. echo.The 'sphinx-build' command was not found. Make sure you have Sphinx echo.installed, then set the SPHINXBUILD environment variable to point echo.to the full path of the 'sphinx-build' executable. Alternatively you echo.may add the Sphinx directory to PATH. echo. echo.If you don't have Sphinx installed, grab it from echo.http://sphinx-doc.org/ exit /b 1 ) %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% goto end :help %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% :end popd ================================================ FILE: docs/requirements.txt ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. setuptools sphinx>=7.0,<8.0 sphinx-reredirects pydata-sphinx-theme>=0.16.1,<0.17.0 myst-parser>=2.0,<4.0 maturin jinja2 ================================================ FILE: docs/source/_static/theme_overrides.css ================================================ /** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ /* Customizing with theme CSS variables */ :root { --pst-color-active-navigation: 215, 70, 51; --pst-color-link-hover: 215, 70, 51; --pst-color-headerlink: 215, 70, 51; /* Use normal text color (like h3, ..) instead of primary color */ --pst-color-h1: var(--color-text-base); --pst-color-h2: var(--color-text-base); /* Use softer blue from bootstrap's default info color */ --pst-color-info: 23, 162, 184; --pst-content-max-width: 100%; /* center column */ --pst-font-size-base: 0.9rem; } /* Scale down the logo in the top navbar */ .bd-header .navbar-brand img.logo__image { height: 40px; width: auto; margin-right: 0.25rem; vertical-align: middle; } /* --- remove the right (secondary) sidebar entirely --- */ .bd-sidebar-secondary { display: none !important; } /* Some versions still reserve the grid column for it — collapse it */ .bd-main { /* left sidebar + content only */ grid-template-columns: 20rem minmax(0, 1fr) !important; } /* --- make the left (primary) sidebar small --- */ .bd-sidebar-primary { width: 20rem !important; flex: 0 0 20rem !important; } /* Optional: make its text a bit tighter */ .bd-sidebar-primary .bd-sidebar { font-size: 0.9rem; } /* --- let the center content use all remaining width --- */ .bd-content, .bd-article-container { max-width: none !important; /* remove internal cap */ } code { color: rgb(215, 70, 51); } .footer { text-align: center; } /* Ensure the logo is properly displayed */ .navbar-brand { height: auto; width: auto; padding: 0 2em; } /* This is the bootstrap CSS style for "table-striped". Since the theme does not yet provide an easy way to configure this globally, it easier to simply include this snippet here than updating each table in all rst files to add ":class: table-striped" */ .table tbody tr:nth-of-type(odd) { background-color: rgba(0, 0, 0, 0.05); } /* Limit the max height of the sidebar navigation section. Because in our customized template, there is more content above the navigation, i.e. larger logo: if we don't decrease the max-height, it will overlap with the footer. Details: 8rem for search box etc*/ @media (min-width:720px) { @supports (position:-webkit-sticky) or (position:sticky) { .bd-links { max-height: calc(100vh - 8rem) } } } /* Fix table text wrapping in RTD theme, * see https://rackerlabs.github.io/docs-rackspace/tools/rtd-tables.html */ @media screen { table.docutils td { /* !important prevents the common CSS stylesheets from overriding this as on RTD they are loaded after this stylesheet */ white-space: normal !important; } } ================================================ FILE: docs/source/_templates/docs-sidebar.html ================================================

================================================ FILE: docs/source/_templates/layout.html ================================================ {% extends "pydata_sphinx_theme/layout.html" %} {% block footer %}
{% for footer_item in theme_footer_items %}
{% include footer_item %}
{% endfor %}

Apache DataFusion, Apache DataFusion Comet, Apache, the Apache feather logo, and the Apache DataFusion project logo

are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.

{% endblock %} ================================================ FILE: docs/source/about/gluten_comparison.md ================================================ # Comparison of Comet and Gluten This document provides a comparison of the Comet and Gluten projects to help guide users who are looking to choose between them. This document is likely biased because the Comet community maintains it. We recommend trying out both Comet and Gluten to see which is the best fit for your needs. This document is based on Comet 0.9.0 and Gluten 1.4.0. ## Architecture Comet and Gluten have very similar architectures. Both are Spark plugins that translate Spark physical plans to a serialized representation and pass the serialized plan to native code for execution. Gluten serializes the plans using the Substrait format and has an extensible architecture that supports execution against multiple engines. Velox and Clickhouse are currently supported, but Velox is more widely used. Comet serializes the plans in a proprietary Protocol Buffer format. Execution is delegated to Apache DataFusion. Comet does not plan to support multiple engines, but rather focus on a tight integration between Spark and DataFusion. ## Underlying Execution Engine: DataFusion vs Velox One of the main differences between Comet and Gluten is the choice of native execution engine. Gluten uses Velox, which is an open-source C++ vectorized query engine created by Meta. Comet uses Apache DataFusion, which is an open-source vectorized query engine implemented in Rust and is governed by the Apache Software Foundation. Velox and DataFusion are both mature query engines that are growing in popularity. From the point of view of the usage of these query engines in Gluten and Comet, the most significant difference is the choice of implementation language (Rust vs C++) and this may be the main factor that users should consider when choosing a solution. For users wishing to implement UDFs in Rust, Comet would likely be a better choice. For users wishing to implement UDFs in C++, Gluten would likely be a better choice. If users are just interested in speeding up their existing Spark jobs and do not need to implement UDFs in native code, then we suggest benchmarking with both solutions and choosing the fastest one for your use case. ![github-stars-datafusion-velox.png](/_static/images/github-stars-datafusion-velox.png) ## Compatibility Comet relies on the full Spark SQL test suite (consisting of more than 24,000 tests) as well its own unit and integration tests to ensure compatibility with Spark. Features that are known to have compatibility differences with Spark are disabled by default, but users can opt in. See the [Comet Compatibility Guide] for more information. [Comet Compatibility Guide]: /user-guide/latest/compatibility.md Gluten also aims to provide compatibility with Spark, and includes a subset of the Spark SQL tests in its own test suite. See the [Gluten Compatibility Guide] for more information. [Gluten Compatibility Guide]: https://apache.github.io/incubator-gluten-site/archives/v1.3.0/velox-backend/limitations/ ## Performance When running a benchmark derived from TPC-H on a single node against local Parquet files, we see that both Comet and Gluten provide an impressive speedup when compared to Spark. Comet provides a 2.4x speedup compares to a 2.8x speedup with Gluten. Gluten is currently faster than Comet for this particular benchmark, but we expect to close that gap over time. Although TPC-H is a good benchmark for operators such as joins and aggregates, it doesn't necessarily represent real-world queries, especially for ETL use cases. For example, there are no complex types involved and no string manipulation, regular expressions, or other advanced expressions. We recommend running your own benchmarks based on your existing Spark jobs. ![tpch_allqueries_comet_gluten.png](/_static/images/tpch_allqueries_comet_gluten.png) The scripts that were used to generate these results can be found [here](https://github.com/apache/datafusion-comet/tree/main/benchmarks/tpc). ## Ease of Development & Contributing Setting up a local development environment with Comet is generally easier than with Gluten due to Rust's package management capabilities vs the complexities around installing C++ dependencies. ## Summary Comet and Gluten are both good solutions for accelerating Spark jobs. We recommend trying both to see which is the best fit for your needs. ================================================ FILE: docs/source/about/index.md ================================================ # Comet Overview Apache DataFusion Comet is a high-performance accelerator for Apache Spark, built on top of the powerful [Apache DataFusion] query engine. Comet is designed to significantly enhance the performance of Apache Spark workloads while leveraging commodity hardware and seamlessly integrating with the Spark ecosystem without requiring any code changes. [Apache DataFusion]: https://datafusion.apache.org The following diagram provides an overview of Comet's architecture. ![Comet Overview](/_static/images/comet-overview.png) ## Architecture The following diagram shows how Comet integrates with Apache Spark. ![Comet System Diagram](/_static/images/comet-system-diagram.png) ## Feature Parity with Apache Spark The project strives to keep feature parity with Apache Spark, that is, users should expect the same behavior (w.r.t features, configurations, query results, etc) with Comet turned on or turned off in their Spark jobs. In addition, Comet extension should automatically detect unsupported features and fallback to Spark engine. ## Comparison with other open-source Spark accelerators There are two other major open-source Spark accelerators: - [Apache Gluten (incubating)](https://github.com/apache/incubator-gluten) - [NVIDIA Spark RAPIDS](https://github.com/NVIDIA/spark-rapids) We have a detailed guide [comparing Apache DataFusion Comet with Apache Gluten]. Spark RAPIDS is a solution that provides hardware acceleration on NVIDIA GPUs. Comet does not require specialized hardware. [comparing Apache DataFusion Comet with Apache Gluten]: gluten_comparison.md ## Getting Started Refer to the [Comet Installation Guide] to get started. [Comet Installation Guide]: /user-guide/latest/installation.md ```{toctree} :maxdepth: 1 :caption: About :hidden: Comparison with Gluten ``` ================================================ FILE: docs/source/asf/index.md ================================================ # ASF Links ```{toctree} :maxdepth: 1 :caption: ASF Links Apache Software Foundation License Donate Thanks Security Code of conduct ``` ================================================ FILE: docs/source/conf.py ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # Configuration file for the Sphinx documentation builder. # # This file only contains a selection of the most common options. For a full # list see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html # -- Path setup -------------------------------------------------------------- # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # # import os # import sys # sys.path.insert(0, os.path.abspath('.')) # -- Project information ----------------------------------------------------- project = 'Apache DataFusion Comet' copyright = '2023-2024, Apache Software Foundation' author = 'Apache Software Foundation' # -- General configuration --------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.autosummary', 'sphinx.ext.doctest', 'sphinx.ext.ifconfig', 'sphinx.ext.mathjax', 'sphinx.ext.viewcode', 'sphinx.ext.napoleon', 'myst_parser', 'sphinx_reredirects', ] source_suffix = { '.rst': 'restructuredtext', '.md': 'markdown', } # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. exclude_patterns = [] # Show members for classes in .. autosummary autodoc_default_options = { "members": None, "undoc-members": None, "show-inheritance": None, "inherited-members": None, } autosummary_generate = True # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # html_theme = 'pydata_sphinx_theme' html_logo = "_static/images/DataFusionComet-Logo-Light.png" html_theme_options = { "logo": { "image_light": "_static/images/DataFusionComet-Logo-Light.png", "image_dark": "_static/images/DataFusionComet-Logo-Dark.png", }, "use_edit_page_button": False, "secondary_sidebar_items": [], "collapse_navigation": True, "navbar_start": ["navbar-logo"], "navbar_center": ["navbar-nav"], "navbar_end": ["navbar-icon-links", "theme-switcher"], "icon_links": [ { "name": "GitHub", "url": "https://github.com/apache/datafusion-comet", "icon": "fa-brands fa-github", }, ], } html_context = { "default_mode": "light", "github_user": "apache", "github_repo": "datafusion-comet", "github_version": "main", "doc_path": "docs/source", } # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] html_css_files = [ "theme_overrides.css" ] html_js_files = [ ("https://buttons.github.io/buttons.js", {'async': 'true', 'defer': 'true'}), ] html_sidebars = { "**": ["docs-sidebar.html"], } # tell myst_parser to auto-generate anchor links for headers h1, h2, h3 myst_heading_anchors = 3 # enable nice rendering of checkboxes for the task lists myst_enable_extensions = ["colon_fence", "deflist", "tasklist"] redirects = { "overview.html": "about/index.html", "gluten_comparison.html": "about/gluten_comparison.html", "user-guide/overview.html": "../about/overview.html", "user-guide/gluten_comparison.html": "../about/gluten_comparison.html", "user-guide/compatibility.html": "latest/compatibility.html", "user-guide/configs.html": "latest/configs.html", "user-guide/datasource.html": "latest/datasource.html", "user-guide/datatypes.html": "latest/datatypes.html", "user-guide/expressions.html": "latest/expressions.html", "user-guide/iceberg.html": "latest/iceberg.html", "user-guide/installation.html": "latest/installation.html", "user-guide/kubernetes.html": "latest/kubernetes.html", "user-guide/metrics.html": "latest/metrics.html", "user-guide/operators.html": "latest/operators.html", "user-guide/source.html": "latest/source.html", "user-guide/tuning.html": "latest/tuning.html", } ================================================ FILE: docs/source/contributor-guide/adding_a_new_expression.md ================================================ # Adding a New Expression There are a number of Spark expression that are not supported by DataFusion Comet yet, and implementing them is a good way to contribute to the project. Before you start, have a look through [these slides](https://docs.google.com/presentation/d/1H0fF2MOkkBK8fPBlnqK6LejUeLcVD917JhVWfp3mb8A/edit#slide=id.p) as they provide a conceptual overview. And a video of a presentation on those slides is available [here](https://drive.google.com/file/d/1POU4lFAZfYwZR8zV1X2eoLiAmc1GDtSP/view?usp=sharing). ## Finding an Expression to Add You may have a specific expression in mind that you'd like to add, but if not, you can review the [expression coverage document](https://github.com/apache/datafusion-comet/blob/main/docs/spark_expressions_support.md) to see which expressions are not yet supported. ## Implementing the Expression Once you have the expression you'd like to add, you should take inventory of the following: 1. What is the Spark expression's behavior across different Spark versions? These make good test cases and will inform you of any compatibility issues, such as an API change that will have to be addressed. 2. Check if the expression is already implemented in DataFusion and if it is compatible with the Spark expression. 1. If it is, you can potentially reuse the existing implementation though you'll need to add tests to verify compatibility. 2. If it's not, consider an initial version in DataFusion for expressions that are common across different engines. For expressions that are specific to Spark, consider an initial version in DataFusion Comet. 3. Test cases for the expression. As mentioned, you can refer to Spark's test cases for a good idea of what to test. Once you know what you want to add, you'll need to update the query planner to recognize the new expression in Scala and potentially add a new expression implementation in the Rust package. ### Adding the Expression in Scala DataFusion Comet uses a framework based on the `CometExpressionSerde` trait for converting Spark expressions to protobuf. Instead of a large match statement, each expression type has its own serialization handler. For aggregate expressions, use the `CometAggregateExpressionSerde` trait instead. #### Creating a CometExpressionSerde Implementation First, create an object that extends `CometExpressionSerde[T]` where `T` is the Spark expression type. This is typically added to one of the serde files in `spark/src/main/scala/org/apache/comet/serde/` (e.g., `math.scala`, `strings.scala`, `arrays.scala`, etc.). For example, the `unhex` function looks like this: ```scala object CometUnhex extends CometExpressionSerde[Unhex] { override def convert( expr: Unhex, inputs: Seq[Attribute], binding: Boolean): Option[ExprOuterClass.Expr] = { val childExpr = exprToProtoInternal(expr.child, inputs, binding) val failOnErrorExpr = exprToProtoInternal(Literal(expr.failOnError), inputs, binding) val optExpr = scalarFunctionExprToProtoWithReturnType( "unhex", expr.dataType, false, childExpr, failOnErrorExpr) optExprWithInfo(optExpr, expr, expr.child) } } ``` The `CometExpressionSerde` trait provides three methods you can override: - `convert(expr: T, inputs: Seq[Attribute], binding: Boolean): Option[Expr]` - **Required**. Converts the Spark expression to protobuf. Return `None` if the expression cannot be converted. - `getSupportLevel(expr: T): SupportLevel` - Optional. Returns the level of support for the expression. See "Using getSupportLevel" section below for details. - `getExprConfigName(expr: T): String` - Optional. Returns a short name for configuration keys. Defaults to the Spark class name. For simple scalar functions that map directly to a DataFusion function, you can use the built-in `CometScalarFunction` implementation: ```scala classOf[Cos] -> CometScalarFunction("cos") ``` #### Registering the Expression Handler Once you've created your `CometExpressionSerde` implementation, register it in `QueryPlanSerde.scala` by adding it to the appropriate expression map (e.g., `mathExpressions`, `stringExpressions`, `predicateExpressions`, etc.): ```scala private val mathExpressions: Map[Class[_ <: Expression], CometExpressionSerde[_]] = Map( // ... other expressions ... classOf[Unhex] -> CometUnhex, classOf[Hex] -> CometHex) ``` The `exprToProtoInternal` method will automatically use this mapping to find and invoke your handler when it encounters the corresponding Spark expression type. A few things to note: - The `convert` method is recursively called on child expressions using `exprToProtoInternal`, so you'll need to make sure that the child expressions are also converted to protobuf. - `scalarFunctionExprToProtoWithReturnType` is for scalar functions that need to return type information. Your expression may use a different method depending on the type of expression. - Use helper methods like `createBinaryExpr` and `createUnaryExpr` from `QueryPlanSerde` for common expression patterns. #### Using getSupportLevel The `getSupportLevel` method allows you to control whether an expression should be executed by Comet based on various conditions such as data types, parameter values, or other expression-specific constraints. This is particularly useful when: 1. Your expression only supports specific data types 2. Your expression has known incompatibilities with Spark's behavior 3. Your expression has edge cases that aren't yet supported The method returns one of three `SupportLevel` values: - **`Compatible(notes: Option[String] = None)`** - Comet supports this expression with full compatibility with Spark, or may have known differences in specific edge cases that are unlikely to be an issue for most users. This is the default if you don't override `getSupportLevel`. - **`Incompatible(notes: Option[String] = None)`** - Comet supports this expression but results can be different from Spark. The expression will only be used if `spark.comet.expr.allowIncompatible=true` or the expression-specific config `spark.comet.expr..allowIncompatible=true` is set. - **`Unsupported(notes: Option[String] = None)`** - Comet does not support this expression under the current conditions. The expression will not be used and Spark will fall back to its native execution. All three support levels accept an optional `notes` parameter to provide additional context about the support level. ##### Examples **Example 1: Restricting to specific data types** The `Abs` expression only supports numeric types: ```scala object CometAbs extends CometExpressionSerde[Abs] { override def getSupportLevel(expr: Abs): SupportLevel = { expr.child.dataType match { case _: NumericType => Compatible() case _ => // Spark supports NumericType, DayTimeIntervalType, and YearMonthIntervalType Unsupported(Some("Only integral, floating-point, and decimal types are supported")) } } override def convert( expr: Abs, inputs: Seq[Attribute], binding: Boolean): Option[ExprOuterClass.Expr] = { // ... conversion logic ... } } ``` **Example 2: Validating parameter values** The `TruncDate` expression only supports specific format strings: ```scala object CometTruncDate extends CometExpressionSerde[TruncDate] { val supportedFormats: Seq[String] = Seq("year", "yyyy", "yy", "quarter", "mon", "month", "mm", "week") override def getSupportLevel(expr: TruncDate): SupportLevel = { expr.format match { case Literal(fmt: UTF8String, _) => if (supportedFormats.contains(fmt.toString.toLowerCase(Locale.ROOT))) { Compatible() } else { Unsupported(Some(s"Format $fmt is not supported")) } case _ => Incompatible( Some("Invalid format strings will throw an exception instead of returning NULL")) } } override def convert( expr: TruncDate, inputs: Seq[Attribute], binding: Boolean): Option[ExprOuterClass.Expr] = { // ... conversion logic ... } } ``` **Example 3: Marking known incompatibilities** The `ArrayAppend` expression has known behavioral differences from Spark: ```scala object CometArrayAppend extends CometExpressionSerde[ArrayAppend] { override def getSupportLevel(expr: ArrayAppend): SupportLevel = Incompatible(None) override def convert( expr: ArrayAppend, inputs: Seq[Attribute], binding: Boolean): Option[ExprOuterClass.Expr] = { // ... conversion logic ... } } ``` This expression will only be used when users explicitly enable incompatible expressions via configuration. ##### How getSupportLevel Affects Execution When the query planner encounters an expression: 1. It first checks if the expression is explicitly disabled via `spark.comet.expr..enabled=false` 2. It then calls `getSupportLevel` on the expression handler 3. Based on the result: - `Compatible()`: Expression proceeds to conversion - `Incompatible()`: Expression is skipped unless `spark.comet.expr.allowIncompatible=true` or expression-specific allow config is set - `Unsupported()`: Expression is skipped and a fallback to Spark occurs Any notes provided will be logged to help with debugging and understanding why an expression was not used. #### Adding Spark-side Tests for the New Expression It is important to verify that the new expression is correctly recognized by the native execution engine and matches the expected Spark behavior. The preferred way to add test coverage is to write a SQL test file using the SQL file test framework. This approach is simpler than writing Scala test code and makes it easy to cover many input combinations and edge cases. ##### Writing a SQL test file Create a `.sql` file under the appropriate subdirectory in `spark/src/test/resources/sql-tests/expressions/` (e.g., `string/`, `math/`, `array/`). The file should create a table with test data, then run queries that exercise the expression. Here is an example for the `unhex` expression: ```sql statement CREATE TABLE test_unhex(col string) USING parquet statement INSERT INTO test_unhex VALUES ('537061726B2053514C'), ('737472696E67'), ('\0'), (''), ('###'), ('G123'), ('hello'), ('A1B'), ('0A1B'), (NULL) -- column argument query SELECT unhex(col) FROM test_unhex -- literal arguments query SELECT unhex('48656C6C6F'), unhex(''), unhex(NULL) ``` Each `query` block automatically runs the SQL through both Spark and Comet and compares results, and also verifies that Comet executes the expression natively (not falling back to Spark). Run the test with: ```shell ./mvnw test -Dsuites="org.apache.comet.CometSqlFileTestSuite unhex" -Dtest=none ``` For full documentation on the test file format — including directives like `ConfigMatrix`, query modes like `spark_answer_only` and `tolerance`, handling known bugs with `ignore(...)`, and tips for writing thorough tests — see the [SQL File Tests](sql-file-tests.md) guide. ##### Tips - **Cover both column references and literals.** Comet often uses different code paths for each. The SQL file test suite automatically disables constant folding, so all-literal queries are evaluated natively. - **Include edge cases** such as `NULL`, empty strings, boundary values, `NaN`, and multibyte UTF-8 characters. - **Keep one file per expression** to make failures easy to locate. ##### Scala tests (alternative) For cases that require programmatic setup or custom assertions beyond what SQL files support, you can also add Scala test cases in `CometExpressionSuite` using the `checkSparkAnswerAndOperator` method: ```scala test("unhex") { val table = "unhex_table" withTable(table) { sql(s"create table $table(col string) using parquet") sql(s"""INSERT INTO $table VALUES |('537061726B2053514C'), |('737472696E67'), |('\\0'), |(''), |('###'), |('G123'), |('hello'), |('A1B'), |('0A1B')""".stripMargin) checkSparkAnswerAndOperator(s"SELECT unhex(col) FROM $table") } } ``` When writing Scala tests with literal values (e.g., `SELECT my_func('literal')`), Spark's constant folding optimizer may evaluate the expression at planning time, bypassing Comet. To prevent this, disable constant folding: ```scala test("my_func with literals") { withSQLConf(SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> "org.apache.spark.sql.catalyst.optimizer.ConstantFolding") { checkSparkAnswerAndOperator("SELECT my_func('literal_value')") } } ``` ### Adding the Expression To the Protobuf Definition Once you have the expression implemented in Scala, you might need to update the protobuf definition to include the new expression. You may not need to do this if the expression is already covered by the existing protobuf definition (e.g. you're adding a new scalar function that uses the `ScalarFunc` message). You can find the protobuf definition in `native/proto/src/proto/expr.proto`, and in particular the `Expr` or potentially the `AggExpr` messages. If you were to add a new expression called `Add2`, you would add a new case to the `Expr` message like so: ```proto message Expr { oneof expr_struct { ... Add2 add2 = 100; // Choose the next available number } } ``` Then you would define the `Add2` message like so: ```proto message Add2 { Expr left = 1; Expr right = 2; } ``` ### Adding the Expression in Rust With the serialization complete, the next step is to implement the expression in Rust and ensure that the incoming plan can make use of it. How this works is somewhat dependent on the type of expression you're adding. Expression implementations live in the `native/spark-expr/src/` directory, organized by category (e.g., `math_funcs/`, `string_funcs/`, `array_funcs/`). #### Generally Adding a New Expression If you're adding a new expression that requires custom protobuf serialization, you may need to: 1. Add a new message to the protobuf definition in `native/proto/src/proto/expr.proto` 2. Add a native expression handler in `expression_registry.rs` to deserialize the new protobuf message type and create a native expression For most expressions, you can skip this step if you're using the existing scalar function infrastructure. #### Adding a New Scalar Function Expression For a new scalar function, you can reuse a lot of code by updating the `create_comet_physical_fun` method in `native/spark-expr/src/comet_scalar_funcs.rs`. Add a match case for your function name: ```rust match fun_name { // ... other functions ... "unhex" => { let func = Arc::new(spark_unhex); make_comet_scalar_udf!("unhex", func, without data_type) } // ... more functions ... } ``` The `make_comet_scalar_udf!` macro has several variants depending on whether your function needs: - A data type parameter: `make_comet_scalar_udf!("ceil", spark_ceil, data_type)` - No data type parameter: `make_comet_scalar_udf!("unhex", func, without data_type)` - An eval mode: `make_comet_scalar_udf!("decimal_div", spark_decimal_div, data_type, eval_mode)` - A fail_on_error flag: `make_comet_scalar_udf!("spark_modulo", func, without data_type, fail_on_error)` #### Implementing the Function Then implement your function in an appropriate module under `native/spark-expr/src/`. The function signature will look like: ```rust pub fn spark_unhex(args: &[ColumnarValue]) -> Result { // Do the work here } ``` If your function uses the data type or eval mode, the signature will include those as additional parameters: ```rust pub fn spark_ceil( args: &[ColumnarValue], data_type: &DataType ) -> Result { // Implementation } ``` ### API Differences Between Spark Versions If the expression you're adding has different behavior across different Spark versions, you'll need to account for that in your implementation. There are two tools at your disposal to help with this: 1. Shims that exist in `spark/src/main/spark-$SPARK_VERSION/org/apache/comet/shims/CometExprShim.scala` for each Spark version. These shims are used to provide compatibility between different Spark versions. 2. Variables that correspond to the Spark version, such as `isSpark33Plus`, which can be used to conditionally execute code based on the Spark version. ## Shimming to Support Different Spark Versions If the expression you're adding has different behavior across different Spark versions, you can use the shim system located in `spark/src/main/spark-$SPARK_VERSION/org/apache/comet/shims/CometExprShim.scala` for each Spark version. The `CometExprShim` trait provides several mechanisms for handling version differences: 1. **Version-specific methods** - Override methods in the trait to provide version-specific behavior 2. **Version-specific expression handling** - Use `versionSpecificExprToProtoInternal` to handle expressions that only exist in certain Spark versions For example, the `StringDecode` expression only exists in certain Spark versions. The shim handles this: ```scala trait CometExprShim { def versionSpecificExprToProtoInternal( expr: Expression, inputs: Seq[Attribute], binding: Boolean): Option[Expr] = { expr match { case s: StringDecode => stringDecode(expr, s.charset, s.bin, inputs, binding) case _ => None } } } ``` The `QueryPlanSerde.exprToProtoInternal` method calls `versionSpecificExprToProtoInternal` first, allowing shims to intercept and handle version-specific expressions before falling back to the standard expression maps. Your `CometExpressionSerde` implementation can also access shim methods by mixing in the `CometExprShim` trait, though in most cases you can directly access the expression properties if they're available across all supported Spark versions. ## Resources - [Variance PR](https://github.com/apache/datafusion-comet/pull/297) - Aggregation function - [Unhex PR](https://github.com/apache/datafusion-comet/pull/342) - Basic scalar function with shims for different Spark versions ================================================ FILE: docs/source/contributor-guide/adding_a_new_operator.md ================================================ # Adding a New Operator This guide explains how to add support for a new Spark physical operator in Apache DataFusion Comet. ## Overview `CometExecRule` is responsible for replacing Spark operators with Comet operators. There are different approaches to implementing Comet operators depending on where they execute and how they integrate with the native execution engine. ### Types of Comet Operators `CometExecRule` maintains two distinct maps of operators: #### 1. Native Operators (`nativeExecs` map) These operators run entirely in native Rust code and are the primary way to accelerate Spark workloads. Native operators are registered in the `nativeExecs` map in `CometExecRule.scala`. Key characteristics of native operators: - They are converted to their corresponding native protobuf representation - They execute as DataFusion operators in the native engine - The `CometOperatorSerde` implementation handles enable/disable checks, support validation, and protobuf serialization Examples: `ProjectExec`, `FilterExec`, `SortExec`, `HashAggregateExec`, `SortMergeJoinExec`, `ExpandExec`, `WindowExec` #### 2. Sink Operators (`sinks` map) Sink operators serve as entry points (data sources) for native execution blocks. They are registered in the `sinks` map in `CometExecRule.scala`. Key characteristics of sinks: - They become `ScanExec` operators in the native plan (see `operator2Proto` in `CometExecRule.scala`) - They can be leaf nodes that feed data into native execution blocks - They are wrapped with `CometScanWrapper` or `CometSinkPlaceHolder` during plan transformation - Examples include operators that bring data from various sources into native execution Examples: `UnionExec`, `CoalesceExec`, `CollectLimitExec`, `TakeOrderedAndProjectExec` Special sinks (not in the `sinks` map but also treated as sinks): - `CometScanExec` - File scans - `CometSparkToColumnarExec` - Conversion from Spark row format - `ShuffleExchangeExec` / `BroadcastExchangeExec` - Exchange operators #### 3. Comet JVM Operators These operators run in the JVM but are part of the Comet execution path. For JVM operators, all checks happen in `CometExecRule` rather than using `CometOperatorSerde`, because they don't need protobuf serialization. Examples: `CometBroadcastExchangeExec`, `CometShuffleExchangeExec` ### Choosing the Right Operator Type When adding a new operator, choose based on these criteria: **Use Native Operators when:** - The operator transforms data (e.g., project, filter, sort, aggregate, join) - The operator has a direct DataFusion equivalent or custom implementation - The operator consumes native child operators and produces native output - The operator is in the middle of an execution pipeline **Use Sink Operators when:** - The operator serves as a data source for native execution (becomes a `ScanExec`) - The operator brings data from non-native sources (e.g., `UnionExec` combining multiple inputs) - The operator is typically a leaf or near-leaf node in the execution tree - The operator needs special handling to interface with the native engine **Implementation Note for Sinks:** Sink operators are handled specially in `CometExecRule.operator2Proto`. Instead of converting to their own operator type, they are converted to `ScanExec` in the native plan. This allows them to serve as entry points for native execution blocks. The original Spark operator is wrapped with `CometScanWrapper` or `CometSinkPlaceHolder` which manages the boundary between JVM and native execution. ## Implementing a Native Operator This section focuses on adding a native operator, which is the most common and complex case. ### Step 1: Define the Protobuf Message First, add the operator definition to `native/proto/src/proto/operator.proto`. #### Add to the Operator Message Add your new operator to the `oneof op_struct` in the main `Operator` message: ```proto message Operator { repeated Operator children = 1; uint32 plan_id = 2; oneof op_struct { Scan scan = 100; Projection projection = 101; Filter filter = 102; // ... existing operators ... YourNewOperator your_new_operator = 112; // Choose next available number } } ``` #### Define the Operator Message Create a message for your operator with the necessary fields: ```proto message YourNewOperator { // Fields specific to your operator repeated spark.spark_expression.Expr expressions = 1; // Add other configuration fields as needed } ``` For reference, see existing operators like `Filter` (simple), `HashAggregate` (complex), or `Sort` (with ordering). ### Step 2: Create a CometOperatorSerde Implementation Create a new Scala file in `spark/src/main/scala/org/apache/comet/serde/operator/` (e.g., `CometYourOperator.scala`) that extends `CometOperatorSerde[T]` where `T` is the Spark operator type. The `CometOperatorSerde` trait provides several key methods: - `enabledConfig: Option[ConfigEntry[Boolean]]` - Configuration to enable/disable this operator - `getSupportLevel(operator: T): SupportLevel` - Determines if the operator is supported - `convert(op: T, builder: Operator.Builder, childOp: Operator*): Option[Operator]` - Converts to protobuf - `createExec(nativeOp: Operator, op: T): CometNativeExec` - Creates the Comet execution operator wrapper The validation workflow in `CometExecRule.isOperatorEnabled`: 1. Checks if the operator is enabled via `enabledConfig` 2. Calls `getSupportLevel()` to determine compatibility 3. Handles Compatible/Incompatible/Unsupported cases with appropriate fallback messages #### Simple Example (Filter) ```scala object CometFilterExec extends CometOperatorSerde[FilterExec] { override def enabledConfig: Option[ConfigEntry[Boolean]] = Some(CometConf.COMET_EXEC_FILTER_ENABLED) override def convert( op: FilterExec, builder: Operator.Builder, childOp: OperatorOuterClass.Operator*): Option[OperatorOuterClass.Operator] = { val cond = exprToProto(op.condition, op.child.output) if (cond.isDefined && childOp.nonEmpty) { val filterBuilder = OperatorOuterClass.Filter .newBuilder() .setPredicate(cond.get) Some(builder.setFilter(filterBuilder).build()) } else { withInfo(op, op.condition, op.child) None } } override def createExec(nativeOp: Operator, op: FilterExec): CometNativeExec = { CometFilterExec(nativeOp, op, op.output, op.condition, op.child, SerializedPlan(None)) } } case class CometFilterExec( override val nativeOp: Operator, override val originalPlan: SparkPlan, override val output: Seq[Attribute], condition: Expression, child: SparkPlan, override val serializedPlanOpt: SerializedPlan) extends CometUnaryExec { override def outputPartitioning: Partitioning = child.outputPartitioning override def outputOrdering: Seq[SortOrder] = child.outputOrdering override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan = this.copy(child = newChild) } ``` #### More Complex Example (Project) ```scala object CometProjectExec extends CometOperatorSerde[ProjectExec] { override def enabledConfig: Option[ConfigEntry[Boolean]] = Some(CometConf.COMET_EXEC_PROJECT_ENABLED) override def convert( op: ProjectExec, builder: Operator.Builder, childOp: Operator*): Option[OperatorOuterClass.Operator] = { val exprs = op.projectList.map(exprToProto(_, op.child.output)) if (exprs.forall(_.isDefined) && childOp.nonEmpty) { val projectBuilder = OperatorOuterClass.Projection .newBuilder() .addAllProjectList(exprs.map(_.get).asJava) Some(builder.setProjection(projectBuilder).build()) } else { withInfo(op, op.projectList: _*) None } } override def createExec(nativeOp: Operator, op: ProjectExec): CometNativeExec = { CometProjectExec(nativeOp, op, op.output, op.projectList, op.child, SerializedPlan(None)) } } case class CometProjectExec( override val nativeOp: Operator, override val originalPlan: SparkPlan, override val output: Seq[Attribute], projectList: Seq[NamedExpression], child: SparkPlan, override val serializedPlanOpt: SerializedPlan) extends CometUnaryExec with PartitioningPreservingUnaryExecNode { override def producedAttributes: AttributeSet = outputSet override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan = this.copy(child = newChild) } ``` #### Using getSupportLevel Override `getSupportLevel` to control operator support based on specific conditions: ```scala override def getSupportLevel(operator: YourOperatorExec): SupportLevel = { // Check for unsupported features if (operator.hasUnsupportedFeature) { return Unsupported(Some("Feature X is not supported")) } // Check for incompatible behavior if (operator.hasKnownDifferences) { return Incompatible(Some("Known differences in edge case Y")) } Compatible() } ``` Support levels: - **`Compatible()`** - Fully compatible with Spark (default) - **`Incompatible()`** - Supported but may differ; requires explicit opt-in - **`Unsupported()`** - Not supported under current conditions Note that Comet will treat an operator as incompatible if any of the child expressions are incompatible. ### Step 3: Register the Operator Add your operator to the appropriate map in `CometExecRule.scala`: #### For Native Operators Add to the `nativeExecs` map (`CometExecRule.scala`): ```scala val nativeExecs: Map[Class[_ <: SparkPlan], CometOperatorSerde[_]] = Map( classOf[ProjectExec] -> CometProjectExec, classOf[FilterExec] -> CometFilterExec, // ... existing operators ... classOf[YourOperatorExec] -> CometYourOperator, ) ``` #### For Sink Operators If your operator is a sink (becomes a `ScanExec` in the native plan), add to the `sinks` map (`CometExecRule.scala`): ```scala val sinks: Map[Class[_ <: SparkPlan], CometOperatorSerde[_]] = Map( classOf[CoalesceExec] -> CometCoalesceExec, classOf[UnionExec] -> CometUnionExec, // ... existing operators ... classOf[YourSinkOperatorExec] -> CometYourSinkOperator, ) ``` Note: The `allExecs` map automatically combines both `nativeExecs` and `sinks`, so you only need to add to one of the two maps. ### Step 4: Add Configuration Entry Add a configuration entry in `common/src/main/scala/org/apache/comet/CometConf.scala`: ```scala val COMET_EXEC_YOUR_OPERATOR_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.exec.yourOperator.enabled") .doc("Whether to enable your operator in Comet") .booleanConf .createWithDefault(true) ``` Run `make` to update the user guide. The new configuration option will be added to `docs/source/user-guide/latest/configs.md`. ### Step 5: Implement the Native Operator in Rust #### Update the Planner In `native/core/src/execution/planner.rs`, add a match case in the operator deserialization logic to handle your new protobuf message: ```rust use datafusion_comet_proto::spark_operator::operator::OpStruct; // In the create_plan or similar method: match op.op_struct.as_ref() { Some(OpStruct::Scan(scan)) => { // ... existing cases ... } Some(OpStruct::YourNewOperator(your_op)) => { create_your_operator_exec(your_op, children, session_ctx) } // ... other cases ... } ``` #### Implement the Operator Create the operator implementation, either in an existing file or a new file in `native/core/src/execution/operators/`: ```rust use datafusion::physical_plan::{ExecutionPlan, ...}; use datafusion_comet_proto::spark_operator::YourNewOperator; pub fn create_your_operator_exec( op: &YourNewOperator, children: Vec>, session_ctx: &SessionContext, ) -> Result, ExecutionError> { // Deserialize expressions and configuration // Create and return the execution plan // Option 1: Use existing DataFusion operator // Ok(Arc::new(SomeDataFusionExec::try_new(...)?)) // Option 2: Implement custom operator (see ExpandExec for example) // Ok(Arc::new(YourCustomExec::new(...))) } ``` For custom operators, you'll need to implement the `ExecutionPlan` trait. See `native/core/src/execution/operators/expand.rs` or `scan.rs` for examples. ### Step 6: Add Tests #### Scala Integration Tests Add tests in `spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala` or a related test suite: ```scala test("your operator") { withTable("test_table") { sql("CREATE TABLE test_table(col1 INT, col2 STRING) USING parquet") sql("INSERT INTO test_table VALUES (1, 'a'), (2, 'b')") // Test query that uses your operator checkSparkAnswerAndOperator( "SELECT * FROM test_table WHERE col1 > 1" ) } } ``` The `checkSparkAnswerAndOperator` helper verifies: 1. Results match Spark's native execution 2. Your operator is actually being used (not falling back) #### Rust Unit Tests Add unit tests in your Rust implementation file: ```rust #[cfg(test)] mod tests { use super::*; #[test] fn test_your_operator() { // Test operator creation and execution } } ``` ### Step 7: Update Documentation Add your operator to the supported operators list in `docs/source/user-guide/latest/compatibility.md` or similar documentation. ## Implementing a Sink Operator Sink operators are converted to `ScanExec` in the native plan and serve as entry points for native execution. The implementation is simpler than native operators because sink operators extend the `CometSink` base class which provides the conversion logic. ### Step 1: Create a CometOperatorSerde Implementation Create a new Scala file in `spark/src/main/scala/org/apache/spark/sql/comet/` (e.g., `CometYourSinkOperator.scala`): ```scala import org.apache.comet.serde.operator.CometSink object CometYourSinkOperator extends CometSink[YourSinkExec] { override def enabledConfig: Option[ConfigEntry[Boolean]] = Some(CometConf.COMET_EXEC_YOUR_SINK_ENABLED) // Optional: Override if the data produced is FFI safe override def isFfiSafe: Boolean = false override def createExec( nativeOp: OperatorOuterClass.Operator, op: YourSinkExec): CometNativeExec = { CometSinkPlaceHolder( nativeOp, op, CometYourSinkExec(op, op.output, /* other parameters */, op.child)) } // Optional: Override getSupportLevel if you need custom validation beyond data types override def getSupportLevel(operator: YourSinkExec): SupportLevel = { // CometSink base class already checks data types in convert() // Add any additional validation here Compatible() } } /** * Comet implementation of YourSinkExec that supports columnar processing */ case class CometYourSinkExec( override val originalPlan: SparkPlan, override val output: Seq[Attribute], /* other parameters */, child: SparkPlan) extends CometExec with UnaryExecNode { override protected def doExecuteColumnar(): RDD[ColumnarBatch] = { // Implement columnar execution logic val rdd = child.executeColumnar() // Apply your sink operator's logic rdd } override def outputPartitioning: Partitioning = { // Define output partitioning } override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan = this.copy(child = newChild) } ``` **Key Points:** - Extend `CometSink[T]` which provides the `convert()` method that transforms the operator to `ScanExec` - The `CometSink.convert()` method (in `CometSink.scala`) automatically handles: - Data type validation - Conversion to `ScanExec` in the native plan - Setting FFI safety flags - You must implement `createExec()` to wrap the operator appropriately - You typically need to create a corresponding `CometYourSinkExec` class that implements columnar execution ### Step 2: Register the Sink Add your sink to the `sinks` map in `CometExecRule.scala`: ```scala val sinks: Map[Class[_ <: SparkPlan], CometOperatorSerde[_]] = Map( classOf[CoalesceExec] -> CometCoalesceExec, classOf[UnionExec] -> CometUnionExec, classOf[YourSinkExec] -> CometYourSinkOperator, ) ``` ### Step 3: Add Configuration Add a configuration entry in `CometConf.scala`: ```scala val COMET_EXEC_YOUR_SINK_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.exec.yourSink.enabled") .doc("Whether to enable your sink operator in Comet") .booleanConf .createWithDefault(true) ``` ### Step 4: Add Tests Test that your sink operator correctly feeds data into native execution: ```scala test("your sink operator") { withTable("test_table") { sql("CREATE TABLE test_table(col1 INT, col2 STRING) USING parquet") sql("INSERT INTO test_table VALUES (1, 'a'), (2, 'b')") // Test query that uses your sink operator followed by native operators checkSparkAnswerAndOperator( "SELECT col1 + 1 FROM (/* query that produces YourSinkExec */)" ) } } ``` **Important Notes for Sinks:** - Sinks extend the `CometSink` base class, which provides the `convert()` method implementation - The `CometSink.convert()` method automatically handles conversion to `ScanExec` in the native plan - You don't need to add protobuf definitions for sink operators - they use the standard `Scan` message - You don't need Rust implementation for sinks - they become standard `ScanExec` operators that read from the JVM - Sink implementations should provide a columnar-compatible execution class (e.g., `CometCoalesceExec`) - The `createExec()` method wraps the operator with `CometSinkPlaceHolder` to manage the JVM-to-native boundary - See `CometCoalesceExec.scala` or `CometUnionExec` in `spark/src/main/scala/org/apache/spark/sql/comet/` for reference implementations ## Implementing a JVM Operator For operators that run in the JVM: 1. Create a new operator class extending appropriate Spark base classes in `spark/src/main/scala/org/apache/comet/` 2. Add matching logic in `CometExecRule.scala` to transform the Spark operator 3. No protobuf or Rust implementation needed Example pattern from `CometExecRule.scala`: ```scala case s: ShuffleExchangeExec if nativeShuffleSupported(s) => CometShuffleExchangeExec(s, shuffleType = CometNativeShuffle) ``` ## Common Patterns and Helpers ### Expression Conversion Use `QueryPlanSerde.exprToProto` to convert Spark expressions to protobuf: ```scala val protoExpr = exprToProto(sparkExpr, inputSchema) ``` ### Handling Fallback Use `withInfo` to tag operators with fallback reasons: ```scala if (!canConvert) { withInfo(op, "Reason for fallback", childNodes: _*) return None } ``` ### Child Operator Validation Always check that child operators were successfully converted: ```scala if (childOp.isEmpty) { // Cannot convert if children failed return None } ``` ## Debugging Tips 1. **Enable verbose logging**: Set `spark.comet.explain.format=verbose` to see detailed plan transformations 2. **Check fallback reasons**: Set `spark.comet.logFallbackReasons.enabled=true` to log why operators fall back to Spark 3. **Verify protobuf**: Add debug prints in Rust to inspect deserialized operators 4. **Use EXPLAIN**: Run `EXPLAIN EXTENDED` on queries to see the physical plan ================================================ FILE: docs/source/contributor-guide/benchmark-results/blaze-0.5.0-tpcds.json ================================================ { "engine": "datafusion-comet", "benchmark": "tpcds", "data_path": "/mnt/bigdata/tpcds/sf100", "query_path": "/mnt/bigdata/tpcds/queries", "spark_conf": { "spark.blaze.forceShuffledHashJoin": "true", "spark.driver.port": "35949", "spark.eventLog.enabled": "true", "spark.sql.analyzer.canonicalization.multiCommutativeOpMemoryOptThreshold": "2147483647", "spark.cores.max": "16", "spark.driver.extraClassPath": "/opt/blaze/blaze-engine-spark-3.5-release-5.0.0-SNAPSHOT.jar", "spark.app.submitTime": "1753976545258", "spark.executor.memory": "16g", "spark.shuffle.manager": "org.apache.spark.sql.execution.blaze.shuffle.BlazeShuffleManager", "spark.executor.instances": "2", "spark.app.startTime": "1753976545559", "spark.serializer.objectStreamReset": "100", "spark.driver.host": "10.0.0.118", "spark.submit.deployMode": "client", "spark.app.name": "blaze benchmark derived from tpcds", "spark.executor.cores": "8", "spark.driver.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false", "spark.executor.memoryOverhead": "16g", "spark.sql.extensions": "org.apache.spark.sql.blaze.BlazeSparkSessionExtension", "spark.blaze.enable": "true", "spark.sql.adaptive.enabled": "true", "spark.executor.id": "driver", "spark.master": "spark://woody:7077", "spark.sql.adaptive.forceApply": "true", "spark.repl.local.jars": "file:///opt/blaze/blaze-engine-spark-3.5-release-5.0.0-SNAPSHOT.jar", "spark.memory.offHeap.enabled": "false", "spark.driver.memory": "8G", "spark.jars": "file:///opt/blaze/blaze-engine-spark-3.5-release-5.0.0-SNAPSHOT.jar", "spark.app.initial.jar.urls": "spark://10.0.0.118:35949/jars/blaze-engine-spark-3.5-release-5.0.0-SNAPSHOT.jar", "spark.app.id": "app-20250731094225-0000", "spark.rdd.compress": "True", "spark.executor.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false", "spark.submit.pyFiles": "", "spark.sql.warehouse.dir": "file:/home/andy/git/apache/datafusion-comet/dev/benchmarks/spark-warehouse", "spark.executor.extraClassPath": "/opt/blaze/blaze-engine-spark-3.5-release-5.0.0-SNAPSHOT.jar" }, "1": [ 4.29833984375 ], "2": [ 3.2578773498535156 ], "3": [ 1.8652851581573486 ], "4": [ 28.0850727558136 ], "5": [ 5.483048439025879 ], "6": [ 1.8989448547363281 ], "7": [ 2.7168705463409424 ], "8": [ 2.0041682720184326 ], "9": [ 6.036026239395142 ], "10": [ 2.158907175064087 ], "11": [ 12.088945388793945 ], "12": [ 1.6278820037841797 ], "13": [ 4.141969919204712 ], "14": [ 27.16408944129944 ], "15": [ 7.972369909286499 ], "16": [ 19.509708166122437 ], "17": [ 2.9771337509155273 ], "18": [ 3.184837818145752 ], "19": [ 3.512895107269287 ], "20": [ 2.5250091552734375 ], "21": [ 2.032010316848755 ], "22": [ 10.054096698760986 ], "23": [ 37.075517892837524 ], "24": [ 4.5132224559783936 ], "25": [ 3.260453939437866 ], "26": [ 1.6490797996520996 ], "27": [ 2.562084436416626 ], "28": [ 6.1220972537994385 ], "29": [ 3.355574607849121 ], "30": [ 1.3987014293670654 ], "31": [ 5.99055290222168 ], "32": [ 1.860522985458374 ], "33": [ 3.1227269172668457 ], "34": [ 1.4920480251312256 ], "35": [ 3.3718972206115723 ], "36": [ 3.137023687362671 ], "37": [ 2.0510008335113525 ], "38": [ 3.81714129447937 ], "39": [ 13.598978281021118 ], "40": [ 3.6551194190979004 ], "41": [ 0.20307397842407227 ], "42": [ 1.4140422344207764 ], "43": [ 1.4719886779785156 ], "44": [ 1.0411145687103271 ], "45": [ 2.1550612449645996 ], "46": [ 4.502093076705933 ], "47": [ 5.857606649398804 ], "48": [ 4.466501235961914 ], "49": [ 4.6020188331604 ], "50": [ 3.786076784133911 ], "51": [ 9.466694355010986 ], "52": [ 1.7045836448669434 ], "53": [ 1.6680505275726318 ], "54": [ 3.2161707878112793 ], "55": [ 1.3395226001739502 ], "56": [ 2.775632619857788 ], "57": [ 2.668205976486206 ], "58": [ 4.912959575653076 ], "59": [ 3.08463454246521 ], "60": [ 2.8756072521209717 ], "61": [ 4.997696161270142 ], "62": [ 1.0771088600158691 ], "63": [ 1.4476990699768066 ], "64": [ 8.340136051177979 ], "65": [ 9.196853160858154 ], "66": [ 3.8863117694854736 ], "67": [ 24.989885330200195 ], "68": [ 4.229811429977417 ], "69": [ 2.1849424839019775 ], "70": [ 3.462109088897705 ], "71": [ 2.9780192375183105 ], "72": [ 98.07282423973083 ], "73": [ 1.2939915657043457 ], "74": [ 17.994235277175903 ], "75": [ 9.07810640335083 ], "76": [ 1.987945795059204 ], "77": [ 3.2832038402557373 ], "78": [ 15.809038877487183 ], "79": [ 2.3230249881744385 ], "80": [ 9.651153087615967 ], "81": [ 1.4056549072265625 ], "82": [ 2.151021718978882 ], "83": [ 0.9110991954803467 ], "84": [ 0.9381260871887207 ], "85": [ 4.502171277999878 ], "86": [ 0.8276948928833008 ], "87": [ 3.5236589908599854 ], "88": [ 7.260309457778931 ], "89": [ 1.6737573146820068 ], "90": [ 0.6564974784851074 ], "91": [ 0.5676999092102051 ], "92": [ 0.9616613388061523 ], "93": [ 4.848900318145752 ], "94": [ 9.96233868598938 ], "95": [ 36.08780312538147 ], "96": [ 0.9135136604309082 ], "97": [ 4.4856956005096436 ], "98": [ 4.316124677658081 ], "99": [ 1.8331043720245361 ] } ================================================ FILE: docs/source/contributor-guide/benchmark-results/blaze-0.5.0-tpch.json ================================================ { "engine": "datafusion-comet", "benchmark": "tpch", "data_path": "/mnt/bigdata/tpch/sf100/", "query_path": "/mnt/bigdata/tpch/queries/", "spark_conf": { "spark.app.name": "blaze benchmark derived from tpch", "spark.blaze.forceShuffledHashJoin": "true", "spark.eventLog.enabled": "true", "spark.app.id": "app-20250731095942-0000", "spark.sql.analyzer.canonicalization.multiCommutativeOpMemoryOptThreshold": "2147483647", "spark.driver.extraClassPath": "/opt/blaze/blaze-engine-spark-3.5-release-5.0.0-SNAPSHOT.jar", "spark.app.submitTime": "1753977581869", "spark.app.initial.jar.urls": "spark://10.0.0.118:34855/jars/blaze-engine-spark-3.5-release-5.0.0-SNAPSHOT.jar", "spark.executor.memory": "16g", "spark.shuffle.manager": "org.apache.spark.sql.execution.blaze.shuffle.BlazeShuffleManager", "spark.serializer.objectStreamReset": "100", "spark.driver.host": "10.0.0.118", "spark.submit.deployMode": "client", "spark.executor.cores": "8", "spark.driver.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false", "spark.executor.memoryOverhead": "16g", "spark.sql.extensions": "org.apache.spark.sql.blaze.BlazeSparkSessionExtension", "spark.blaze.enable": "true", "spark.sql.adaptive.enabled": "true", "spark.executor.id": "driver", "spark.master": "spark://woody:7077", "spark.sql.adaptive.forceApply": "true", "spark.repl.local.jars": "file:///opt/blaze/blaze-engine-spark-3.5-release-5.0.0-SNAPSHOT.jar", "spark.memory.offHeap.enabled": "false", "spark.driver.memory": "8G", "spark.jars": "file:///opt/blaze/blaze-engine-spark-3.5-release-5.0.0-SNAPSHOT.jar", "spark.driver.port": "34855", "spark.rdd.compress": "True", "spark.executor.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false", "spark.app.startTime": "1753977582179", "spark.executor.instances": "1", "spark.cores.max": "8", "spark.submit.pyFiles": "", "spark.sql.warehouse.dir": "file:/home/andy/git/apache/datafusion-comet/dev/benchmarks/spark-warehouse", "spark.executor.extraClassPath": "/opt/blaze/blaze-engine-spark-3.5-release-5.0.0-SNAPSHOT.jar" }, "1": [ 9.673133850097656 ], "2": [ 5.968065500259399 ], "3": [ 11.941619157791138 ], "4": [ 11.886268138885498 ], "5": [ 23.566248416900635 ], "6": [ 2.833730697631836 ], "7": [ 13.405823469161987 ], "8": [ 23.069715976715088 ], "9": [ 39.501715898513794 ], "10": [ 11.25059962272644 ], "11": [ 4.793748140335083 ], "12": [ 6.490489959716797 ], "13": [ 6.752679824829102 ], "14": [ 3.9720609188079834 ], "15": [ 8.393324375152588 ], "16": [ 2.637575149536133 ], "17": [ 29.3641459941864 ], "18": [ 35.25342917442322 ], "19": [ 6.869648456573486 ], "20": [ 7.14982533454895 ], "21": [ 51.66389608383179 ], "22": [ 4.223745107650757 ] } ================================================ FILE: docs/source/contributor-guide/benchmark-results/gluten-1.4.0-tpcds.json ================================================ { "engine": "datafusion-comet", "benchmark": "tpcds", "data_path": "/mnt/bigdata/tpcds/sf100", "query_path": "/mnt/bigdata/tpcds/queries", "spark_conf": { "spark.eventLog.enabled": "true", "spark.gluten.sql.session.timeZone.default": "UTC", "spark.plugins": "org.apache.gluten.GlutenPlugin", "spark.app.name": "gluten benchmark derived from tpcds", "spark.cores.max": "16", "spark.gluten.memoryOverhead.size.in.bytes": "5153751040", "spark.memory.offHeap.enabled": "true", "spark.repl.local.jars": "file:///opt/gluten/gluten-velox-bundle-spark3.5_2.12-linux_amd64-1.4.0.jar", "spark.executor.extraClassPath": "/opt/gluten/gluten-velox-bundle-spark3.5_2.12-linux_amd64-1.4.0.jar", "spark.executor.instances": "2", "spark.executor.memory": "16G", "spark.driver.extraClassPath": "/opt/gluten/gluten-velox-bundle-spark3.5_2.12-linux_amd64-1.4.0.jar", "spark.driver.port": "38403", "spark.driver.host": "10.0.0.118", "spark.serializer.objectStreamReset": "100", "spark.app.submitTime": "1753671747082", "spark.submit.deployMode": "client", "spark.app.id": "app-20250728030228-0000", "spark.executor.cores": "8", "spark.app.initial.jar.urls": "spark://10.0.0.118:38403/jars/gluten-velox-bundle-spark3.5_2.12-linux_amd64-1.4.0.jar", "spark.jars": "file:///opt/gluten/gluten-velox-bundle-spark3.5_2.12-linux_amd64-1.4.0.jar", "spark.driver.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false", "spark.gluten.memory.offHeap.size.in.bytes": "17179869184", "spark.shuffle.manager": "org.apache.spark.shuffle.sort.ColumnarShuffleManager", "spark.executor.id": "driver", "spark.master": "spark://woody:7077", "spark.executor.memoryOverhead": "4915", "spark.sql.extensions": "org.apache.gluten.extension.GlutenSessionExtensions", "spark.gluten.numTaskSlotsPerExecutor": "8", "spark.driver.memory": "8G", "spark.gluten.memory.task.offHeap.size.in.bytes": "2147483648", "spark.memory.offHeap.size": "17179869184", "spark.rdd.compress": "True", "spark.executor.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false", "spark.app.startTime": "1753671747382", "spark.gluten.memory.conservative.task.offHeap.size.in.bytes": "1073741824", "spark.gluten.sql.columnar.forceShuffledHashJoin": "true", "spark.sql.adaptive.customCostEvaluatorClass": "org.apache.spark.sql.execution.adaptive.GlutenCostEvaluator", "spark.submit.pyFiles": "", "spark.gluten.memory.dynamic.offHeap.sizing.enabled": "false", "spark.sql.session.timeZone": "UTC", "spark.sql.warehouse.dir": "file:/home/andy/git/apache/datafusion-comet/dev/benchmarks/spark-warehouse" }, "1": [ 3.0712015628814697 ], "2": [ 3.0666799545288086 ], "3": [ 1.0671255588531494 ], "4": [ 17.276039361953735 ], "5": [ 4.336325407028198 ], "6": [ 2.240464925765991 ], "7": [ 2.1070384979248047 ], "8": [ 2.2761263847351074 ], "9": [ 3.4592885971069336 ], "10": [ 2.2161333560943604 ], "11": [ 8.601247549057007 ], "12": [ 1.3066213130950928 ], "13": [ 2.623806953430176 ], "14": [ 23.665879487991333 ], "15": [ 2.3319854736328125 ], "16": [ 6.435860633850098 ], "17": [ 3.3381738662719727 ], "18": [ 2.4940454959869385 ], "19": [ 3.214183807373047 ], "20": [ 0.8629763126373291 ], "21": [ 1.1388742923736572 ], "22": [ 13.283171892166138 ], "23": [ 33.25598192214966 ], "24": [ 13.960798025131226 ], "25": [ 3.493340492248535 ], "26": [ 1.003934383392334 ], "27": [ 1.8391008377075195 ], "28": [ 4.88685941696167 ], "29": [ 3.245540142059326 ], "30": [ 1.6746954917907715 ], "31": [ 7.37675404548645 ], "32": [ 1.4505951404571533 ], "33": [ 2.6308183670043945 ], "34": [ 1.872574806213379 ], "35": [ 2.4917962551116943 ], "36": [ 2.046686887741089 ], "37": [ 0.798964262008667 ], "38": [ 3.3161089420318604 ], "39": [ 4.972580671310425 ], "40": [ 3.0317022800445557 ], "41": [ 0.292935848236084 ], "42": [ 0.9157257080078125 ], "43": [ 1.17030668258667 ], "44": [ 0.5907480716705322 ], "45": [ 1.6078379154205322 ], "46": [ 3.1738409996032715 ], "47": [ 3.6137521266937256 ], "48": [ 2.1740095615386963 ], "49": [ 4.5014050006866455 ], "50": [ 3.800581216812134 ], "51": [ 5.458707571029663 ], "52": [ 0.9787416458129883 ], "53": [ 1.2668261528015137 ], "54": [ 4.599064111709595 ], "55": [ 0.9526393413543701 ], "56": [ 2.744847059249878 ], "57": [ 2.3258957862854004 ], "58": [ 2.707801580429077 ], "59": [ 2.9688880443573 ], "60": [ 2.8286445140838623 ], "61": [ 7.713818073272705 ], "62": [ 1.3007659912109375 ], "63": [ 1.6228349208831787 ], "64": [ 10.753086805343628 ], "65": [ 5.098642826080322 ], "66": [ 1.919985055923462 ], "67": [ 31.3416805267334 ], "68": [ 2.760091781616211 ], "69": [ 2.1082839965820312 ], "70": [ 3.231919050216675 ], "71": [ 3.1762571334838867 ], "72": [ 811.9890258312225 ], "73": [ 1.1076979637145996 ], "74": [ 6.423391580581665 ], "75": [ 6.315303325653076 ], "76": [ 3.0667858123779297 ], "77": [ 2.4472310543060303 ], "78": [ 9.947726488113403 ], "79": [ 2.0369961261749268 ], "80": [ 7.902697801589966 ], "81": [ 1.4782285690307617 ], "82": [ 1.0187582969665527 ], "83": [ 1.3076961040496826 ], "84": [ 0.7356717586517334 ], "85": [ 2.1237404346466064 ], "86": [ 1.196605920791626 ], "87": [ 3.310823917388916 ], "88": [ 5.557470083236694 ], "89": [ 1.4879262447357178 ], "90": [ 1.3077824115753174 ], "91": [ 1.2596931457519531 ], "92": [ 1.1216387748718262 ], "93": [ 3.8458316326141357 ], "94": [ 3.957663059234619 ], "95": [ 18.828344583511353 ], "96": [ 0.8511285781860352 ], "97": [ 3.0931687355041504 ], "98": [ 2.2251434326171875 ], "99": [ 1.0991463661193848 ] } ================================================ FILE: docs/source/contributor-guide/benchmark-results/gluten-1.4.0-tpch.json ================================================ { "engine": "datafusion-comet", "benchmark": "tpch", "data_path": "/mnt/bigdata/tpch/sf100/", "query_path": "/mnt/bigdata/tpch/queries/", "spark_conf": { "spark.eventLog.enabled": "true", "spark.gluten.sql.session.timeZone.default": "UTC", "spark.app.submitTime": "1752337239601", "spark.plugins": "org.apache.gluten.GlutenPlugin", "spark.gluten.memoryOverhead.size.in.bytes": "5153751040", "spark.memory.offHeap.enabled": "true", "spark.repl.local.jars": "file:///opt/gluten/gluten-velox-bundle-spark3.5_2.12-linux_amd64-1.4.0.jar", "spark.executor.extraClassPath": "/opt/gluten/gluten-velox-bundle-spark3.5_2.12-linux_amd64-1.4.0.jar", "spark.driver.port": "46545", "spark.executor.memory": "16G", "spark.driver.extraClassPath": "/opt/gluten/gluten-velox-bundle-spark3.5_2.12-linux_amd64-1.4.0.jar", "spark.serializer.objectStreamReset": "100", "spark.driver.host": "10.0.0.118", "spark.submit.deployMode": "client", "spark.executor.cores": "8", "spark.jars": "file:///opt/gluten/gluten-velox-bundle-spark3.5_2.12-linux_amd64-1.4.0.jar", "spark.driver.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false", "spark.gluten.memory.offHeap.size.in.bytes": "17179869184", "spark.shuffle.manager": "org.apache.spark.shuffle.sort.ColumnarShuffleManager", "spark.app.id": "app-20250712162041-0000", "spark.executor.id": "driver", "spark.master": "spark://woody:7077", "spark.app.name": "gluten benchmark derived from tpch", "spark.executor.memoryOverhead": "4915", "spark.app.startTime": "1752337239956", "spark.sql.extensions": "org.apache.gluten.extension.GlutenSessionExtensions", "spark.gluten.numTaskSlotsPerExecutor": "8", "spark.driver.memory": "8G", "spark.gluten.memory.task.offHeap.size.in.bytes": "2147483648", "spark.memory.offHeap.size": "17179869184", "spark.app.initial.jar.urls": "spark://10.0.0.118:46545/jars/gluten-velox-bundle-spark3.5_2.12-linux_amd64-1.4.0.jar", "spark.rdd.compress": "True", "spark.executor.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false", "spark.gluten.memory.conservative.task.offHeap.size.in.bytes": "1073741824", "spark.gluten.sql.columnar.forceShuffledHashJoin": "true", "spark.sql.adaptive.customCostEvaluatorClass": "org.apache.spark.sql.execution.adaptive.GlutenCostEvaluator", "spark.submit.pyFiles": "", "spark.executor.instances": "1", "spark.cores.max": "8", "spark.sql.session.timeZone": "UTC", "spark.sql.warehouse.dir": "file:/home/andy/git/apache/datafusion-comet/dev/benchmarks/spark-warehouse", "spark.gluten.memory.dynamic.offHeap.sizing.enabled": "false" }, "1": [ 7.653544187545776 ], "2": [ 4.837377309799194 ], "3": [ 7.874017000198364 ], "4": [ 7.201406717300415 ], "5": [ 15.82085108757019 ], "6": [ 1.7845516204833984 ], "7": [ 10.334960460662842 ], "8": [ 16.844615697860718 ], "9": [ 24.79072380065918 ], "10": [ 8.695719480514526 ], "11": [ 3.6076815128326416 ], "12": [ 4.583712339401245 ], "13": [ 7.016078233718872 ], "14": [ 3.003683567047119 ], "15": [ 7.852885007858276 ], "16": [ 3.1703848838806152 ], "17": [ 19.128700017929077 ], "18": [ 20.703757524490356 ], "19": [ 3.294731378555298 ], "20": [ 6.121671199798584 ], "21": [ 41.27772641181946 ], "22": [ 3.542634963989258 ] } ================================================ FILE: docs/source/contributor-guide/benchmark-results/spark-3.5.3-tpcds.json ================================================ { "engine": "datafusion-comet", "benchmark": "tpcds", "data_path": "/mnt/bigdata/tpcds/sf100/", "query_path": "/mnt/bigdata/tpcds/queries/", "spark_conf": { "spark.eventLog.enabled": "true", "spark.driver.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false", "spark.cores.max": "16", "spark.app.submitTime": "1760657346549", "spark.app.startTime": "1760657346845", "spark.hadoop.fs.s3a.aws.credentials.provider": "com.amazonaws.auth.DefaultAWSCredentialsProviderChain", "spark.driver.port": "41151", "spark.master": "spark://woody:7077", "spark.executor.id": "driver", "spark.memory.offHeap.enabled": "true", "spark.driver.memory": "8G", "spark.executor.memory": "16g", "spark.hadoop.fs.s3a.impl": "org.apache.hadoop.fs.s3a.S3AFileSystem", "spark.app.id": "app-20251016172907-0000", "spark.rdd.compress": "True", "spark.executor.instances": "2", "spark.executor.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false", "spark.serializer.objectStreamReset": "100", "spark.submit.pyFiles": "", "spark.submit.deployMode": "client", "spark.sql.warehouse.dir": "file:/home/andy/git/apache/datafusion-comet/dev/benchmarks/spark-warehouse", "spark.app.name": "spark benchmark derived from tpcds", "spark.executor.cores": "8", "spark.driver.host": "woody.lan", "spark.memory.offHeap.size": "16g" }, "1": [ 4.523299217224121 ], "2": [ 3.061577081680298 ], "3": [ 1.5242812633514404 ], "4": [ 50.726277589797974 ], "5": [ 12.996135711669922 ], "6": [ 1.9649882316589355 ], "7": [ 3.0779569149017334 ], "8": [ 2.091928720474243 ], "9": [ 5.696160078048706 ], "10": [ 2.6276841163635254 ], "11": [ 14.976734399795532 ], "12": [ 1.0809345245361328 ], "13": [ 3.9019551277160645 ], "14": [ 40.51576566696167 ], "15": [ 3.2053213119506836 ], "16": [ 12.394548416137695 ], "17": [ 4.5201404094696045 ], "18": [ 3.710134506225586 ], "19": [ 2.388089895248413 ], "20": [ 1.1710665225982666 ], "21": [ 1.8241875171661377 ], "22": [ 22.95406436920166 ], "23": [ 91.60468244552612 ], "24": [ 14.566667556762695 ], "25": [ 4.0827412605285645 ], "26": [ 1.9840991497039795 ], "27": [ 2.4599902629852295 ], "28": [ 8.550125122070312 ], "29": [ 4.786677122116089 ], "30": [ 1.974135398864746 ], "31": [ 5.949168920516968 ], "32": [ 1.8300220966339111 ], "33": [ 3.0961456298828125 ], "34": [ 1.5340814590454102 ], "35": [ 4.07404637336731 ], "36": [ 2.289029121398926 ], "37": [ 4.415144681930542 ], "38": [ 7.2151618003845215 ], "39": [ 8.983118057250977 ], "40": [ 8.090219020843506 ], "41": [ 0.2758638858795166 ], "42": [ 1.4861207008361816 ], "43": [ 1.4047019481658936 ], "44": [ 1.1311781406402588 ], "45": [ 1.7742793560028076 ], "46": [ 2.6378352642059326 ], "47": [ 5.033041954040527 ], "48": [ 7.473065137863159 ], "49": [ 7.34236478805542 ], "50": [ 10.566041946411133 ], "51": [ 13.90496039390564 ], "52": [ 1.2180531024932861 ], "53": [ 1.8595950603485107 ], "54": [ 2.967790365219116 ], "55": [ 1.1428017616271973 ], "56": [ 2.6576313972473145 ], "57": [ 3.1256778240203857 ], "58": [ 3.110593795776367 ], "59": [ 3.6817147731781006 ], "60": [ 2.342132568359375 ], "61": [ 3.5699379444122314 ], "62": [ 1.0830762386322021 ], "63": [ 1.5091686248779297 ], "64": [ 21.24241876602173 ], "65": [ 11.174801588058472 ], "66": [ 2.968890428543091 ], "67": [ 56.65302085876465 ], "68": [ 2.7817704677581787 ], "69": [ 2.238774061203003 ], "70": [ 2.8807759284973145 ], "71": [ 2.878760576248169 ], "72": [ 87.73793363571167 ], "73": [ 1.3357255458831787 ], "74": [ 17.069884300231934 ], "75": [ 12.418950080871582 ], "76": [ 2.7246549129486084 ], "77": [ 3.2758727073669434 ], "78": [ 31.570815801620483 ], "79": [ 2.1316466331481934 ], "80": [ 25.685710191726685 ], "81": [ 2.383108615875244 ], "82": [ 2.1166131496429443 ], "83": [ 1.825127124786377 ], "84": [ 1.4185044765472412 ], "85": [ 4.894887924194336 ], "86": [ 0.9917283058166504 ], "87": [ 7.2068891525268555 ], "88": [ 5.049949645996094 ], "89": [ 1.8566343784332275 ], "90": [ 0.8218870162963867 ], "91": [ 1.0735762119293213 ], "92": [ 1.3357479572296143 ], "93": [ 21.765851259231567 ], "94": [ 6.729715585708618 ], "95": [ 23.40083408355713 ], "96": [ 0.6852657794952393 ], "97": [ 11.261420965194702 ], "98": [ 2.003811836242676 ], "99": [ 1.5077838897705078 ] } ================================================ FILE: docs/source/contributor-guide/benchmark-results/spark-3.5.3-tpch.json ================================================ { "engine": "datafusion-comet", "benchmark": "tpch", "data_path": "/mnt/bigdata/tpch/sf100/", "query_path": "/mnt/bigdata/tpch/queries/", "spark_conf": { "spark.app.submitTime": "1760654734485", "spark.eventLog.enabled": "true", "spark.driver.port": "45867", "spark.driver.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false", "spark.executor.id": "driver", "spark.hadoop.fs.s3a.aws.credentials.provider": "com.amazonaws.auth.DefaultAWSCredentialsProviderChain", "spark.app.startTime": "1760654734783", "spark.master": "spark://woody:7077", "spark.memory.offHeap.enabled": "true", "spark.driver.memory": "8G", "spark.executor.memory": "16g", "spark.hadoop.fs.s3a.impl": "org.apache.hadoop.fs.s3a.S3AFileSystem", "spark.rdd.compress": "True", "spark.app.name": "spark benchmark derived from tpch", "spark.executor.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false", "spark.serializer.objectStreamReset": "100", "spark.executor.instances": "1", "spark.cores.max": "8", "spark.app.id": "app-20251016164535-0000", "spark.submit.pyFiles": "", "spark.submit.deployMode": "client", "spark.sql.warehouse.dir": "file:/home/andy/git/apache/datafusion-comet/dev/benchmarks/spark-warehouse", "spark.executor.cores": "8", "spark.driver.host": "woody.lan", "spark.memory.offHeap.size": "16g" }, "1": [ 82.14922451972961 ], "2": [ 13.39389944076538 ], "3": [ 26.29530620574951 ], "4": [ 22.093881368637085 ], "5": [ 52.291497230529785 ], "6": [ 3.5577659606933594 ], "7": [ 23.285813808441162 ], "8": [ 36.04274129867554 ], "9": [ 77.91567945480347 ], "10": [ 20.83555793762207 ], "11": [ 13.609780311584473 ], "12": [ 13.592298984527588 ], "13": [ 22.524098873138428 ], "14": [ 6.217005252838135 ], "15": [ 16.401182413101196 ], "16": [ 7.700634241104126 ], "17": [ 70.78637218475342 ], "18": [ 77.76704359054565 ], "19": [ 7.267014503479004 ], "20": [ 10.419305562973022 ], "21": [ 72.9061770439148 ], "22": [ 9.621031522750854 ] } ================================================ FILE: docs/source/contributor-guide/benchmark-results/tpc-ds.md ================================================ # Apache DataFusion Comet: Benchmarks Derived From TPC-DS The following benchmarks were performed on a Linux workstation with PCIe 5, AMD 7950X CPU (16 cores), 128 GB RAM, and data stored locally in Parquet format on NVMe storage. Performance characteristics will vary in different environments and we encourage you to run these benchmarks in your own environments. The operating system is Ubuntu 22.04.5 LTS. The tracking issue for improving TPC-DS performance is [#858](https://github.com/apache/datafusion-comet/issues/858). ![](../../_static/images/benchmark-results/0.11.0/tpcds_allqueries.png) Here is a breakdown showing relative performance of Spark and Comet for each query. ![](../../_static/images/benchmark-results/0.11.0/tpcds_queries_compare.png) The following chart shows how much Comet currently accelerates each query from the benchmark in relative terms. ![](../../_static/images/benchmark-results/0.11.0/tpcds_queries_speedup_rel.png) The following chart shows how much Comet currently accelerates each query from the benchmark in absolute terms. ![](../../_static/images/benchmark-results/0.11.0/tpcds_queries_speedup_abs.png) The raw results of these benchmarks in JSON format is available here: - [Spark](spark-3.5.3-tpcds.json) - [Comet](comet-0.11.0-tpcds.json) The scripts that were used to generate these results can be found [here](https://github.com/apache/datafusion-comet/tree/main/benchmarks/tpc). ================================================ FILE: docs/source/contributor-guide/benchmark-results/tpc-h.md ================================================ # Apache DataFusion Comet: Benchmarks Derived From TPC-H The following benchmarks were performed on a Linux workstation with PCIe 5, AMD 7950X CPU (16 cores), 128 GB RAM, and data stored locally in Parquet format on NVMe storage. Performance characteristics will vary in different environments and we encourage you to run these benchmarks in your own environments. The operating system is Ubuntu 22.04.5 LTS. The tracking issue for improving TPC-H performance is [#391](https://github.com/apache/datafusion-comet/issues/391). ![](../../_static/images/benchmark-results/0.11.0/tpch_allqueries.png) Here is a breakdown showing relative performance of Spark and Comet for each query. ![](../../_static/images/benchmark-results/0.11.0/tpch_queries_compare.png) The following chart shows how much Comet currently accelerates each query from the benchmark in relative terms. ![](../../_static/images/benchmark-results/0.11.0/tpch_queries_speedup_rel.png) The following chart shows how much Comet currently accelerates each query from the benchmark in absolute terms. ![](../../_static/images/benchmark-results/0.11.0/tpch_queries_speedup_abs.png) The raw results of these benchmarks in JSON format is available here: - [Spark](spark-3.5.3-tpch.json) - [Comet](comet-0.11.0-tpch.json) The scripts that were used to generate these results can be found [here](https://github.com/apache/datafusion-comet/tree/main/benchmarks/tpc). ================================================ FILE: docs/source/contributor-guide/benchmarking.md ================================================ # Comet Benchmarking Guide To track progress on performance, we regularly run benchmarks derived from TPC-H and TPC-DS. The benchmarking scripts are contained [here](https://github.com/apache/datafusion-comet/tree/main/benchmarks/tpc). Data generation scripts are available in the [DataFusion Benchmarks](https://github.com/apache/datafusion-benchmarks) GitHub repository. ## Current Benchmark Results The published benchmarks are performed on a Linux workstation with PCIe 5, AMD 7950X CPU (16 cores), 128 GB RAM, and data stored locally in Parquet format on NVMe storage. Performance characteristics will vary in different environments and we encourage you to run these benchmarks in your own environments. The operating system used was Ubuntu 22.04.5 LTS. - [Benchmarks derived from TPC-H](benchmark-results/tpc-h) - [Benchmarks derived from TPC-DS](benchmark-results/tpc-ds) ## Benchmarking Guides Available benchmarking guides: - [Benchmarking on macOS](benchmarking_macos.md) - [Benchmarking on AWS EC2](benchmarking_aws_ec2) - [TPC-DS Benchmarking with spark-sql-perf](benchmarking_spark_sql_perf.md) We also have many micro benchmarks that can be run from an IDE located [here](https://github.com/apache/datafusion-comet/tree/main/spark/src/test/scala/org/apache/spark/sql/benchmark). ================================================ FILE: docs/source/contributor-guide/benchmarking_aws_ec2.md ================================================ # Comet Benchmarking in EC2 This guide is for setting up benchmarks on AWS EC2 with a single node with Parquet files either located on an attached EBS volume, or stored in S3. ## Create EC2 Instance - Create an EC2 instance with an EBS volume sized for approximately 2x the size of the dataset to be generated (200 GB for scale factor 100, 2 TB for scale factor 1000, and so on) - Optionally, create an S3 bucket to store the Parquet files Recommendation: Use `c7i.4xlarge` instance type with `provisioned iops 2` EBS volume (8000 IOPS). ## Install Prerequisites ```shell sudo yum update -y sudo yum install -y git protobuf-compiler java-17-amazon-corretto-headless java-17-amazon-corretto-devel sudo yum groupinstall -y "Development Tools" export JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto ``` ## Generate Benchmark Data ```shell curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh RUSTFLAGS='-C target-cpu=native' cargo install tpchgen-cli tpchgen-cli -s 100 --format parquet --parts 32 --output-dir data ``` Rename the generated directories so that they have a `.parquet` suffix. For example, rename `customer` to `customer.parquet`. Set the `TPCH_DATA` environment variable. This will be referenced by the benchmarking scripts. ```shell export TPCH_DATA=/home/ec2-user/data ``` ## Install Apache Spark ```shell export SPARK_VERSION=3.5.6 wget https://archive.apache.org/dist/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop3.tgz tar xzf spark-$SPARK_VERSION-bin-hadoop3.tgz sudo mv spark-$SPARK_VERSION-bin-hadoop3 /opt export SPARK_HOME=/opt/spark-$SPARK_VERSION-bin-hadoop3/ mkdir /tmp/spark-events ``` Set `SPARK_MASTER` env var (IP address will need to be edited): ```shell export SPARK_MASTER=spark://172.31.34.87:7077 ``` Set `SPARK_LOCAL_DIRS` to point to EBS volume ```shell sudo mkdir /mnt/tmp sudo chmod 777 /mnt/tmp mv $SPARK_HOME/conf/spark-env.sh.template $SPARK_HOME/conf/spark-env.sh ``` Add the following entry to `spark-env.sh`: ```shell SPARK_LOCAL_DIRS=/mnt/tmp ``` ## Git Clone DataFusion Repositories ```shell git clone https://github.com/apache/datafusion-benchmarks.git git clone https://github.com/apache/datafusion-comet.git ``` Build Comet ```shell cd datafusion-comet make release ``` Set `COMET_JAR` environment variable. ```shell export COMET_JAR=/home/ec2-user/datafusion-comet/spark/target/comet-spark-spark3.5_2.12-$COMET_VERSION.jar ``` ## Run Benchmarks Use the scripts in `benchmarks/tpc` in the Comet repository. ```shell cd benchmarks/tpc export TPCH_QUERIES=/home/ec2-user/datafusion-benchmarks/tpch/queries/ ``` Run Spark benchmark: ```shell python3 run.py --engine spark --benchmark tpch ``` Run Comet benchmark: ```shell python3 run.py --engine comet --benchmark tpch ``` ## Running Benchmarks with S3 Copy the Parquet data to an S3 bucket. ```shell aws s3 cp /home/ec2-user/data s3://your-bucket-name/--recursive ``` Update `TPCH_DATA` environment variable. ```shell export TPCH_DATA=s3a://your-bucket-name ``` Install Hadoop jar files: ```shell wget https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.4/hadoop-aws-3.3.4.jar -P $SPARK_HOME/jars wget https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.11.1026/aws-java-sdk-bundle-1.11.1026.jar -P $SPARK_HOME/jars ``` Add credentials to `~/.aws/credentials`: ```shell [default] aws_access_key_id=your-access-key aws_secret_access_key=your-secret-key ``` Modify the scripts to add the following configurations. ```shell --conf spark.hadoop.fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem \ --conf spark.hadoop.fs.s3a.aws.credentials.provider=com.amazonaws.auth.DefaultAWSCredentialsProviderChain \ ``` Now run the benchmarks: ```shell python3 run.py --engine spark --benchmark tpch python3 run.py --engine comet --benchmark tpch ``` ================================================ FILE: docs/source/contributor-guide/benchmarking_macos.md ================================================ # Comet Benchmarking on macOS This guide is for setting up TPC-H benchmarks locally on macOS using the 100 GB dataset. Note that running this benchmark on macOS is not ideal because we cannot force Spark or Comet to use performance cores rather than efficiency cores, and background processes are sharing these cores. Also, power and thermal management may throttle CPU cores. ## Prerequisites Java and Rust must be installed locally. ## Data Generation ```shell cargo install tpchgen-cli mkdir benchmark_data cd benchmark_data tpchgen-cli -s 100 --format=parquet export $BENCH_DATA=`pwd` ``` Create a temp folder for spark events emitted during benchmarking ```shell mkdir /tmp/spark-events ``` ## Clone the DataFusion Benchmarks Repository ```shell git clone https://github.com/apache/datafusion-benchmarks.git cd export DF_BENCH=`pwd` ``` ## Install Spark Install Apache Spark. This example refers to 3.5.4 version. ```shell wget https://archive.apache.org/dist/spark/spark-3.5.4/spark-3.5.4-bin-hadoop3.tgz tar xzf spark-3.5.4-bin-hadoop3.tgz sudo mv spark-3.5.4-bin-hadoop3 /opt export SPARK_HOME=/opt/spark-3.5.4-bin-hadoop3/ ``` Start Spark in standalone mode: ```shell $SPARK_HOME/sbin/start-master.sh ``` Set `SPARK_MASTER` env var (host name will need to be edited): ```shell export SPARK_MASTER=spark://Rustys-MacBook-Pro.local:7077 ``` ```shell $SPARK_HOME/sbin/start-worker.sh $SPARK_MASTER ``` ### Start local Apache Spark cluster using `spark-class` For Apache Spark distributions installed using `brew` tool, it may happen there is no `$SPARK_HOME/sbin` folder on your machine. In order to start local Apache Spark cluster on `localhost:7077` port, run: ```shell $SPARK_HOME/bin/spark-class org.apache.spark.deploy.master.Master --host 127.0.0.1 --port 7077 --webui-port 8080 ``` Once master has started, in separate console start the worker referring the spark master uri on `localhost:7077` ```shell $SPARK_HOME/bin/spark-class org.apache.spark.deploy.worker.Worker --cores 8 --memory 16G spark://localhost:7077 ``` ## Run Spark Benchmarks Run the following command (the `--data` parameter will need to be updated to point to your TPC-H data): ```shell $SPARK_HOME/bin/spark-submit \ --master $SPARK_MASTER \ --conf spark.driver.memory=8G \ --conf spark.executor.instances=1 \ --conf spark.executor.cores=8 \ --conf spark.cores.max=8 \ --conf spark.executor.memory=16g \ --conf spark.memory.offHeap.enabled=true \ --conf spark.memory.offHeap.size=16g \ --conf spark.eventLog.enabled=true \ $DF_BENCH/runners/datafusion-comet/tpcbench.py \ --benchmark tpch \ --data $BENCH_DATA/tpch-data/ \ --queries $DF_BENCH/tpch/queries \ --output . \ --iterations 1 ``` ## Run Comet Benchmarks Build Comet from source, with `mimalloc` enabled. ```shell make release COMET_FEATURES=mimalloc ``` Set `COMET_JAR` to point to the location of the Comet jar file. Example for Comet 0.8 ```shell export COMET_JAR=`pwd`/spark/target/comet-spark-spark3.5_2.12-0.8.0-SNAPSHOT.jar ``` Run the following command (the `--data` parameter will need to be updated to point to your S3 bucket): ```shell $SPARK_HOME/bin/spark-submit \ --master $SPARK_MASTER \ --conf spark.driver.memory=8G \ --conf spark.executor.instances=1 \ --conf spark.executor.cores=8 \ --conf spark.cores.max=8 \ --conf spark.executor.memory=16g \ --conf spark.memory.offHeap.enabled=true \ --conf spark.memory.offHeap.size=16g \ --conf spark.eventLog.enabled=true \ --jars $COMET_JAR \ --driver-class-path $COMET_JAR \ --conf spark.driver.extraClassPath=$COMET_JAR \ --conf spark.executor.extraClassPath=$COMET_JAR \ --conf spark.plugins=org.apache.spark.CometPlugin \ --conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \ --conf spark.comet.enabled=true \ --conf spark.comet.exec.shuffle.enableFastEncoding=true \ --conf spark.comet.exec.shuffle.fallbackToColumnar=true \ --conf spark.comet.exec.replaceSortMergeJoin=true \ --conf spark.comet.expression.allowIncompatible=true \ $DF_BENCH/runners/datafusion-comet/tpcbench.py \ --benchmark tpch \ --data $BENCH_DATA/tpch-data/ \ --queries $DF_BENCH/tpch/queries \ --output . \ --iterations 1 ``` ================================================ FILE: docs/source/contributor-guide/benchmarking_spark_sql_perf.md ================================================ # TPC-DS Benchmarking with spark-sql-perf This guide explains how to generate TPC-DS data and run TPC-DS benchmarks using the [KubedAI/spark-sql-perf](https://github.com/KubedAI/spark-sql-perf) framework (a fork of [databricks/spark-sql-perf](https://github.com/databricks/spark-sql-perf)). We use the KubedAI fork because it adds two features not present in the upstream Databricks repository: - **Apache Iceberg support** — allows generating and registering TPC-DS tables in Iceberg format, which is needed for Comet Iceberg benchmarking. - **TPC-DS v4.0 queries** — adds the full set of TPC-DS v4.0 queries alongside the existing v1.4 and v2.4 sets. The spark-sql-perf approach uses the TPC-DS `dsdgen` tool to generate data directly through Spark, which handles partitioning and writing to Parquet format automatically. ## Prerequisites - Java 17 (for Spark 3.5+) - Apache Spark 3.5.x - SBT (Scala Build Tool) - C compiler toolchain (`gcc`, `make`, `flex`, `bison`, `byacc`) ## Step 1: Build tpcds-kit The `dsdgen` tool from [databricks/tpcds-kit](https://github.com/databricks/tpcds-kit) is required for data generation. This is a modified fork of the official TPC-DS toolkit that outputs to stdout, allowing Spark to ingest the data directly. **Linux (Ubuntu/Debian):** ```shell sudo apt-get install -y gcc make flex bison byacc git git clone https://github.com/databricks/tpcds-kit.git cd tpcds-kit/tools make OS=LINUX ``` **Linux (CentOS/RHEL/Amazon Linux):** ```shell sudo yum install -y gcc make flex bison byacc git git clone https://github.com/databricks/tpcds-kit.git cd tpcds-kit/tools make OS=LINUX ``` **macOS:** ```shell xcode-select --install git clone https://github.com/databricks/tpcds-kit.git cd tpcds-kit/tools make OS=MACOS ``` Verify the build succeeded: ```shell ls -l dsdgen ``` ## Step 2: Build spark-sql-perf ```shell git clone https://github.com/KubedAI/spark-sql-perf.git cd spark-sql-perf git checkout support-iceberg-tpcds-v4.0 sbt package ``` This produces a JAR file at `target/scala-2.12/spark-sql-perf_2.12-0.5.1-SNAPSHOT.jar` (the exact version may vary). Note the path to this JAR for later use. ## Step 3: Install and Start Spark If you do not already have Spark installed: ```shell export SPARK_VERSION=3.5.6 wget https://archive.apache.org/dist/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop3.tgz tar xzf spark-$SPARK_VERSION-bin-hadoop3.tgz sudo mv spark-$SPARK_VERSION-bin-hadoop3 /opt export SPARK_HOME=/opt/spark-$SPARK_VERSION-bin-hadoop3/ export JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64 mkdir -p /tmp/spark-events ``` Start Spark in standalone mode: ```shell $SPARK_HOME/sbin/start-master.sh export SPARK_MASTER=spark://$(hostname):7077 $SPARK_HOME/sbin/start-worker.sh $SPARK_MASTER ``` ## Step 4: Generate TPC-DS Data Launch `spark-shell` with the spark-sql-perf JAR loaded: ```shell $SPARK_HOME/bin/spark-shell \ --master $SPARK_MASTER \ --jars /path/to/spark-sql-perf/target/scala-2.12/spark-sql-perf_2.12-0.5.1-SNAPSHOT.jar \ --conf spark.driver.memory=8G \ --conf spark.executor.instances=1 \ --conf spark.executor.cores=8 \ --conf spark.executor.memory=16g ``` In the Spark shell, run the following to generate data. Adjust `scaleFactor` and paths as needed: ```scala import com.databricks.spark.sql.perf.tpcds.TPCDSTables val tpcdsKit = "/path/to/tpcds-kit/tools" val scaleFactor = "100" // 100 GB val dataDir = "/path/to/tpcds-data" val format = "parquet" val numPartitions = 32 // adjust based on cluster size val tables = new TPCDSTables(spark.sqlContext, dsdgenDir = tpcdsKit, scaleFactor = scaleFactor) tables.genData( location = dataDir, format = format, overwrite = true, partitionTables = true, clusterByPartitionColumns = true, filterOutNullPartitionValues = false, numPartitions = numPartitions ) ``` Data generation for SF100 typically takes 20-60 minutes depending on hardware. When complete, exit the shell: ```scala :quit ``` Verify the data was generated: ```shell ls /path/to/tpcds-data/ ``` You should see directories for each TPC-DS table (`store_sales`, `catalog_sales`, `web_sales`, `customer`, `date_dim`, etc.). Set the `TPCDS_DATA` environment variable: ```shell export TPCDS_DATA=/path/to/tpcds-data ``` ## Step 5: Run TPC-DS Benchmarks ### Register Tables Launch `spark-shell` with the spark-sql-perf JAR (same as Step 4) and register the generated data as tables: ```scala import com.databricks.spark.sql.perf.tpcds.{TPCDS, TPCDSTables} val scaleFactor = "100" val dataDir = "/path/to/tpcds-data" val format = "parquet" val databaseName = "tpcds" // Create database and register tables sql(s"CREATE DATABASE IF NOT EXISTS $databaseName") val tables = new TPCDSTables(spark.sqlContext, dsdgenDir = "", scaleFactor = scaleFactor) tables.createExternalTables( location = dataDir, format = format, databaseName = databaseName, overwrite = true, discoverPartitions = true ) sql(s"USE $databaseName") ``` ### Run Spark Baseline ```scala val tpcds = new TPCDS(spark.sqlContext) // Choose a query set: tpcds1_4Queries, tpcds2_4Queries, or tpcds4_0Queries val queries = tpcds.tpcds2_4Queries val experiment = tpcds.runExperiment( executionsToRun = queries, iterations = 3, resultLocation = "/path/to/results/spark", tags = Map("engine" -> "spark", "scale_factor" -> "100"), forkThread = true ) experiment.waitForFinish(86400) ``` Results are saved as JSON to the `resultLocation` path. ### Run with Comet Build Comet from source and launch `spark-shell` with both the Comet and spark-sql-perf JARs: ```shell make release export COMET_JAR=$(pwd)/spark/target/comet-spark-spark3.5_2.12-*.jar $SPARK_HOME/bin/spark-shell \ --master $SPARK_MASTER \ --jars /path/to/spark-sql-perf/target/scala-2.12/spark-sql-perf_2.12-0.5.1-SNAPSHOT.jar,$COMET_JAR \ --conf spark.driver.memory=8G \ --conf spark.executor.instances=1 \ --conf spark.executor.cores=8 \ --conf spark.executor.memory=8g \ --conf spark.memory.offHeap.enabled=true \ --conf spark.memory.offHeap.size=8g \ --conf spark.executor.extraClassPath=$COMET_JAR \ --conf spark.sql.extensions=org.apache.comet.CometSparkSessionExtensions \ --conf spark.comet.enabled=true \ --conf spark.comet.exec.enabled=true \ --conf spark.comet.exec.all.enabled=true \ --conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \ --conf spark.comet.exec.shuffle.enabled=true \ --conf spark.comet.columnar.shuffle.enabled=true ``` Then register tables and run the benchmark the same way as the Spark baseline, changing the tags and result location: ```scala val experiment = tpcds.runExperiment( executionsToRun = queries, iterations = 3, resultLocation = "/path/to/results/comet", tags = Map("engine" -> "comet", "scale_factor" -> "100"), forkThread = true ) experiment.waitForFinish(86400) ``` ### View Results Results are saved as JSON under the `resultLocation`. You can query them directly in Spark: ```scala val results = spark.read.json("/path/to/results/spark") results.select("name", "parsingTime", "analysisTime", "optimizationTime", "planningTime", "executionTime") .withColumn("totalTime", (col("parsingTime") + col("analysisTime") + col("optimizationTime") + col("planningTime") + col("executionTime")) / 1000.0) .orderBy("name") .show(200, false) ``` ## Alternative: Command-Line Data Generation You can also generate TPC-DS data without the Spark shell using `spark-submit`: ```shell $SPARK_HOME/bin/spark-submit \ --master $SPARK_MASTER \ --class com.databricks.spark.sql.perf.tpcds.GenTPCDSData \ --conf spark.driver.memory=8G \ --conf spark.executor.instances=1 \ --conf spark.executor.cores=8 \ --conf spark.executor.memory=16g \ /path/to/spark-sql-perf/target/scala-2.12/spark-sql-perf_2.12-0.5.1-SNAPSHOT.jar \ -d /path/to/tpcds-kit/tools \ -s 100 \ -l /path/to/tpcds-data \ -f parquet ``` ## Troubleshooting ### dsdgen not found Ensure `tpcds-kit/tools/dsdgen` exists and is executable. The `dsdgenDir` parameter in the Spark shell (or `-d` flag in the CLI) must point to the directory containing the `dsdgen` binary, not the binary itself. ### Out of memory during data generation For large scale factors (SF1000+), increase executor memory and the number of partitions: ```shell --conf spark.executor.memory=32g ``` And in the Spark shell, use a higher `numPartitions` value (e.g., 200+). ================================================ FILE: docs/source/contributor-guide/bug_triage.md ================================================ # Bug Triage Guide This guide describes how we prioritize and triage bugs in the Comet project. The goal is to ensure that the most impactful bugs — especially correctness issues that produce wrong results — are identified and addressed before less critical issues. ## Priority Labels Every bug should have exactly one priority label. When filing or triaging a bug, apply the appropriate label from the table below. | Label | Color | Description | Examples | | ------------------- | ------ | ------------------------------------------------------------------------------------ | --------------------------------------------------------------------- | | `priority:critical` | Red | Data corruption, silent wrong results, security vulnerabilities | Wrong aggregation results, FFI data corruption, incorrect cast output | | `priority:high` | Orange | Crashes, panics, segfaults, major functional breakage affecting production workloads | Native engine panic, JVM segfault, NPE on supported code path | | `priority:medium` | Yellow | Functional bugs, performance regressions, broken features that have workarounds | Missing expression support, writer feature gaps, excessive spilling | | `priority:low` | Green | Minor issues, test-only failures, tooling, CI flakes, cosmetic issues | Flaky CI test, build script edge case, documentation generator bug | ### How to Choose a Priority Use this decision tree: 1. **Can this bug cause silent wrong results?** If yes → `priority:critical`. These are the most dangerous bugs because users may not notice the incorrect output. 2. **Does this bug crash the JVM or native engine?** If yes → `priority:high`. Crashes are disruptive but at least visible to the user. 3. **Does this bug break a feature or cause significant performance degradation?** If yes → `priority:medium`. The user can work around it (e.g., falling back to Spark) but it impacts the value of Comet. 4. **Everything else** → `priority:low`. Test failures, CI issues, tooling, and cosmetic problems. ### Escalation Triggers A bug should be escalated to a higher priority if: - A `priority:high` crash is discovered to also produce wrong results silently in some cases → escalate to `priority:critical` - A `priority:medium` bug is reported by multiple users or affects a common workload → consider escalating to `priority:high` - A `priority:low` CI flake is blocking PR merges consistently → escalate to `priority:medium` ## Area Labels Area labels indicate which subsystem is affected. A bug may have multiple area labels. These help contributors find bugs in their area of expertise. | Label | Description | | ------------------ | ----------------------------------------- | | `area:writer` | Native writer (Parquet and other formats) | | `area:shuffle` | Shuffle (JVM and native) | | `area:aggregation` | Hash aggregates, aggregate expressions | | `area:scan` | Data source scan (Parquet, CSV, Iceberg) | | `area:expressions` | Expression evaluation | | `area:ffi` | Arrow FFI / JNI boundary | | `area:ci` | CI/CD, GitHub Actions, build tooling | The following pre-existing labels also serve as area indicators: `native_datafusion`, `native_iceberg_compat`, `spark 4`, `spark sql tests`. ## Triage Process Every new issue is automatically labeled with `requires-triage` when it is opened. This makes it easy to find issues that have not yet been triaged by filtering on that label. Once an issue has been triaged, remove the `requires-triage` label and apply the appropriate priority and area labels. ### For New Issues When a new bug is filed: 1. **Reproduce or verify** the issue if possible. If the report lacks reproduction steps, ask the reporter for more details. 2. **Assess correctness impact first.** Ask: "Could this produce wrong results silently?" This is more important than whether it crashes. 3. **Apply a priority label** using the decision tree above. 4. **Apply area labels** to indicate the affected subsystem(s). 5. **Apply `good first issue`** if the fix is likely straightforward and well-scoped. 6. **Remove the `requires-triage` label** to indicate triage is complete. ### For Existing Bugs Periodically review open bugs to ensure priorities are still accurate: - Has a `priority:medium` bug been open for a long time with user reports? Consider escalating. - Has a `priority:high` bug been fixed by a related change? Close it. - Are there clusters of related bugs that should be tracked under an EPIC? ### Prioritization Principles 1. **Correctness over crashes.** A bug that silently returns wrong results is worse than one that crashes, because crashes are at least visible. 2. **User-reported over test-only.** A bug hit by a real user on a real workload takes priority over one found only in test suites. 3. **Core path over experimental.** Bugs in the default scan mode (`native_comet`) or widely-used expressions take priority over bugs in experimental features like `native_datafusion` or `native_iceberg_compat`. 4. **Production safety over feature completeness.** Fixing a data corruption bug is more important than adding support for a new expression. ## Common Bug Categories ### Correctness Bugs (`priority:critical`) These are bugs where Comet produces different results than Spark without any error or warning. Examples include: - Incorrect cast behavior (e.g., negative zero to string) - Aggregate functions ignoring configuration (e.g., `ignoreNulls`) - Data corruption in FFI boundary (e.g., boolean arrays with non-zero offset) - Type mismatches between partial and final aggregation stages When fixing correctness bugs, always add a regression test that verifies the output matches Spark. ### Crash Bugs (`priority:high`) These are bugs where the native engine panics, segfaults, or throws an unhandled exception. Common patterns include: - **All-scalar inputs:** Some expressions assume at least one columnar input and panic when all inputs are literals (e.g., when `ConstantFolding` is disabled) - **Type mismatches:** Downcasting to the wrong Arrow array type - **Memory safety:** FFI boundary issues, unaligned arrays, GlobalRef lifecycle ### Aggregate Planning Bugs Several bugs relate to how Comet plans hash aggregates across stage boundaries. The key issue is that Spark's AQE may materialize a Comet partial aggregate but then run the final aggregate in Spark (or vice versa), and the intermediate formats may not be compatible. See the [EPIC #2892](https://github.com/apache/datafusion-comet/issues/2892) for the full picture. ### Native Writer Bugs The native Parquet writer has a cluster of known test failures tracked as individual issues (#3417–#3430). These are lower priority since the native writer is still maturing, but they should be addressed before the writer is promoted to production-ready status. ## How to Help with Triage Triage is a valuable contribution that doesn't require writing code. You can help by: - Reviewing new issues and suggesting a priority label - Reproducing reported bugs and adding details - Identifying duplicate issues - Linking related issues together - Testing whether old bugs have been fixed by recent changes ================================================ FILE: docs/source/contributor-guide/contributing.md ================================================ # Contributing to Apache DataFusion Comet We welcome contributions to Comet in many areas, and encourage new contributors to get involved. Here are some areas where you can help: - Testing Comet with existing Spark jobs and reporting issues for any bugs or performance issues - Contributing code to support Spark expressions, operators, and data types that are not currently supported - Reviewing pull requests and helping to test new features for correctness and performance - Improving documentation ## Finding issues to work on We maintain a list of good first issues in GitHub [here](https://github.com/apache/datafusion-comet/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22). We also have a [roadmap](roadmap.md). To assign yourself an issue, comment `take` on the issue. To unassign yourself, comment `untake`. ## Reporting issues We use [GitHub issues](https://github.com/apache/datafusion-comet/issues) for bug reports and feature requests. ## Asking for Help The Comet project uses the same Slack and Discord channels as the main Apache DataFusion project. See details at [Apache DataFusion Communications]. There are dedicated Comet channels in both Slack and Discord. ## Regular public meetings The Comet contributors hold regular video calls where new and current contributors are welcome to ask questions and coordinate on issues that they are working on. See the [Apache DataFusion Comet community meeting] Google document for more information. [Apache DataFusion Communications]: https://datafusion.apache.org/contributor-guide/communication.html [Apache DataFusion Comet community meeting]: https://docs.google.com/document/d/1NBpkIAuU7O9h8Br5CbFksDhX-L9TyO9wmGLPMe0Plc8/edit?usp=sharing ================================================ FILE: docs/source/contributor-guide/debugging.md ================================================ # Comet Debugging Guide This HOWTO describes how to debug JVM code and Native code concurrently. The guide assumes you have: 1. IntelliJ as the Java IDE 2. CLion as the Native IDE. For Rust code, the CLion Rust language plugin is required. Note that the IntelliJ Rust plugin is not sufficient. 3. CLion/LLDB as the native debugger. CLion ships with a bundled LLDB and the Rust community has its own packaging of LLDB (`lldb-rust`). Both provide a better display of Rust symbols than plain LLDB or the LLDB that is bundled with XCode. We will use the LLDB packaged with CLion for this guide. 4. We will use a Comet _unit_ test as the canonical use case. _Caveat: The steps here have only been tested with JDK 11_ on Mac (M1) ## Debugging for Advanced Developers Add a `.lldbinit` to comet/native. This is not strictly necessary but will be useful if you want to use advanced `lldb` debugging. For example, we can ignore some exceptions and signals that are not relevant to our debugging and would otherwise cause the debugger to stop. ``` settings set platform.plugin.darwin.ignored-exceptions EXC_BAD_ACCESS|EXC_BAD_INSTRUCTION process handle -n true -p true -s false SIGBUS SIGSEGV SIGILL ``` ### In IntelliJ 1. Set a breakpoint in `NativeBase.load()`, at a point _after_ the Comet library has been loaded. 1. Add a Debug Configuration for the unit test 1. In the Debug Configuration for that unit test add `-Xint` as a JVM parameter. This option is undocumented _magic_. Without this, the LLDB debugger hits a EXC_BAD_ACCESS (or EXC_BAD_INSTRUCTION) from which one cannot recover. 1. Add a println to the unit test to print the PID of the JVM process. (jps can also be used but this is less error prone if you have multiple jvm processes running) ```scala println("Waiting for Debugger: PID - ", ManagementFactory.getRuntimeMXBean().getName()) ``` This will print something like : `PID@your_machine_name`. For JDK9 and newer ```scala println("Waiting for Debugger: PID - ", ProcessHandle.current.pid) ``` ==> Note the PID 1. Debug-run the test in IntelliJ and wait for the breakpoint to be hit ### In CLion 1. After the breakpoint is hit in IntelliJ, in Clion (or LLDB from terminal or editor) - 1. Attach to the jvm process (make sure the PID matches). In CLion, this is `Run -> Attach to process` 1. Put your breakpoint in the native code 1. Go back to IntelliJ and resume the process. 1. Most debugging in CLion is similar to IntelliJ. For advanced LLDB based debugging the LLDB command line can be accessed from the LLDB tab in the Debugger view. Refer to the [LLDB manual](https://lldb.llvm.org/use/tutorial.html) for LLDB commands. ### After your debugging is done 1. In CLion, detach from the process if not already detached 2. In IntelliJ, the debugger might have lost track of the process. If so, the debugger tab will show the process as running (even if the test/job is shown as completed). 3. Close the debugger tab, and if the IDS asks whether it should terminate the process, click Yes. 4. In terminal, use jps to identify the process with the process id you were debugging. If it shows up as running, kill -9 [pid]. If that doesn't remove the process, don't bother, the process will be left behind as a zombie and will consume no (significant) resources. Eventually it will be cleaned up when you reboot possibly after a software update. ### Additional Info OpenJDK mailing list on debugging the JDK on MacOS Detecting the debugger ). ## Verbose debug ### Enabling Native Backtraces By default, Comet does not show native backtraces when exceptions happen in native code: ```scala scala> spark.sql("my_failing_query").show(false) 24/03/05 17:00:07 ERROR Executor: Exception in task 0.0 in stage 0.0 (TID 0)/ 1] org.apache.comet.CometNativeException: Internal error: MIN/MAX is not expected to receive scalars of incompatible types (Date32("NULL"), Int32(15901)). This was likely caused by a bug in DataFusion's code and we would welcome that you file an bug report in our issue tracker at org.apache.comet.Native.executePlan(Native Method) at org.apache.comet.CometExecIterator.executeNative(CometExecIterator.scala:65) at org.apache.comet.CometExecIterator.getNextBatch(CometExecIterator.scala:111) at org.apache.comet.CometExecIterator.hasNext(CometExecIterator.scala:126) ``` Comet can be built with DataFusion's [backtrace] feature enabled, which will include native back traces in `CometNativeException`. [backtrace]: https://arrow.apache.org/datafusion/user-guide/example-usage.html#enable-backtraces To build Comet with this feature enabled: ```shell make release COMET_FEATURES=backtrace ``` Set `RUST_BACKTRACE=1` for the Spark worker/executor process, or for `spark-submit` if running in local mode. ```console RUST_BACKTRACE=1 $SPARK_HOME/spark-shell --jars spark/target/comet-spark-spark3.5_2.12-$COMET_VERSION.jar --conf spark.plugins=org.apache.spark.CometPlugin --conf spark.comet.enabled=true --conf spark.comet.exec.enabled=true ``` Get the expanded exception details ```scala scala> spark.sql("my_failing_query").show(false) 24/03/05 17:00:49 ERROR Executor: Exception in task 0.0 in stage 0.0 (TID 0) org.apache.comet.CometNativeException: Internal error: MIN/MAX is not expected to receive scalars of incompatible types (Date32("NULL"), Int32(15901)) backtrace: 0: std::backtrace::Backtrace::create 1: datafusion::physical_expr::aggregate::min_max::min 2: ::update_batch 3: as futures_core::stream::Stream>::poll_next 4: comet::execution::jni_api::Java_org_apache_comet_Native_executePlan::{{closure}} 5: _Java_org_apache_comet_Native_executePlan (reduced) This was likely caused by a bug in DataFusion's code and we would welcome that you file an bug report in our issue tracker at org.apache.comet.Native.executePlan(Native Method) at org.apache.comet.CometExecIterator.executeNative(CometExecIterator.scala:65) at org.apache.comet.CometExecIterator.getNextBatch(CometExecIterator.scala:111) at org.apache.comet.CometExecIterator.hasNext(CometExecIterator.scala:126) (reduced) ``` Note: - The backtrace coverage in DataFusion is still improving. So there is a chance the error still not covered, if so feel free to file a [ticket](https://github.com/apache/arrow-datafusion/issues) - The backtrace evaluation comes with performance cost and intended mostly for debugging purposes ### Native log configuration By default, Comet emits native-side logs at the `INFO` level to `stderr`. You can use the `COMET_LOG_LEVEL` environment variable to specify the log level. Supported values are: `OFF`, `ERROR`, `WARN`, `INFO`, `DEBUG`, `TRACE`. For example, to configure native logs at the `DEBUG` level on spark executor: ``` spark.executorEnv.COMET_LOG_LEVEL=DEBUG ``` This produces output like the following: ``` 25/09/15 20:17:42 INFO core/src/lib.rs: Comet native library version 0.11.0 initialized 25/09/15 20:17:44 DEBUG /xxx/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/datafusion-execution-49.0.2/src/disk_manager.rs: Created local dirs [TempDir { path: "/private/var/folders/4p/9gtjq1s10fd6frkv9kzy0y740000gn/T/blockmgr-ba524f95-a792-4d79-b49c-276ba324941e/datafusion-qrpApx" }] as DataFusion working directory 25/09/15 20:17:44 DEBUG /xxx/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/datafusion-functions-nested-49.0.2/src/lib.rs: Overwrite existing UDF: array_to_string 25/09/15 20:17:44 DEBUG /xxx/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/datafusion-functions-nested-49.0.2/src/lib.rs: Overwrite existing UDF: string_to_array 25/09/15 20:17:44 DEBUG /xxx/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/datafusion-functions-nested-49.0.2/src/lib.rs: Overwrite existing UDF: range ... ``` Additionally, you can place a `log4rs.yaml` configuration file inside the Comet configuration directory specified by the `COMET_CONF_DIR` environment variable to enable more advanced logging configurations. This file uses the [log4rs YAML configuration format](https://docs.rs/log4rs/latest/log4rs/#configuration-via-a-yaml-file). For example, see: [log4rs.yaml](https://github.com/apache/datafusion-comet/blob/main/conf/log4rs.yaml). ### Debugging Memory Reservations Set `spark.comet.debug.memory=true` to log all calls that grow or shrink memory reservations. Example log output: ``` [Task 486] MemoryPool[ExternalSorter[6]].try_grow(256232960) returning Ok [Task 486] MemoryPool[ExternalSorter[6]].try_grow(256375168) returning Ok [Task 486] MemoryPool[ExternalSorter[6]].try_grow(256899456) returning Ok [Task 486] MemoryPool[ExternalSorter[6]].try_grow(257296128) returning Ok [Task 486] MemoryPool[ExternalSorter[6]].try_grow(257820416) returning Err [Task 486] MemoryPool[ExternalSorterMerge[6]].shrink(10485760) [Task 486] MemoryPool[ExternalSorter[6]].shrink(150464) [Task 486] MemoryPool[ExternalSorter[6]].shrink(146688) [Task 486] MemoryPool[ExternalSorter[6]].shrink(137856) [Task 486] MemoryPool[ExternalSorter[6]].shrink(141952) [Task 486] MemoryPool[ExternalSorterMerge[6]].try_grow(0) returning Ok [Task 486] MemoryPool[ExternalSorterMerge[6]].try_grow(0) returning Ok [Task 486] MemoryPool[ExternalSorter[6]].shrink(524288) [Task 486] MemoryPool[ExternalSorterMerge[6]].try_grow(0) returning Ok [Task 486] MemoryPool[ExternalSorterMerge[6]].try_grow(68928) returning Ok ``` When backtraces are enabled (see earlier section) then backtraces will be included for failed allocations. ================================================ FILE: docs/source/contributor-guide/development.md ================================================ # Comet Development Guide ## Project Layout ``` ├── common <- common Java/Scala code ├── conf <- configuration files ├── native <- native code, in Rust ├── spark <- Spark integration ``` ## Threading Architecture Comet's native execution runs on a shared tokio multi-threaded runtime. Understanding this architecture is important because it affects how you write native operators and JVM callbacks. ### How execution works Spark calls into native code via JNI from an **executor task thread**. There are two execution paths depending on whether the plan reads data from the JVM: **Async I/O path (no JVM data sources, e.g. Iceberg scans):** The DataFusion stream is spawned onto a tokio worker thread and batches are delivered to the executor thread via an `mpsc` channel. The executor thread parks in `blocking_recv()` until the next batch is ready. This avoids busy-polling on I/O-bound workloads. **JVM data source path (ScanExec present):** The executor thread calls `block_on()` and polls the DataFusion stream directly, interleaving `pull_input_batches()` calls on `Poll::Pending` to feed data from the JVM into ScanExec operators. In both cases, DataFusion operators execute on **tokio worker threads**, not on the Spark executor task thread. All Spark tasks on an executor share one tokio runtime. ### Rules for native code **Do not use `thread_local!` or assume thread identity.** Tokio may run your operator's `poll` method on any worker thread, and may move it between threads across polls. Any state must live in the operator struct or be shared via `Arc`. **JNI calls work from any thread, but have overhead.** `JVMClasses::get_env()` calls `AttachCurrentThread`, which acquires JVM internal locks. The `AttachGuard` detaches the thread when dropped. Repeated attach/detach cycles on tokio workers add overhead, so avoid calling into the JVM on hot paths during stream execution. **Do not call `TaskContext.get()` from JVM callbacks during execution.** Spark's `TaskContext` is a `ThreadLocal` on the executor task thread. JVM methods invoked from tokio worker threads will see `null`. If you need task metadata, capture it at construction time (in `createPlan` or operator setup) and store it in the operator. See `CometTaskMemoryManager` for an example — it captures `TaskContext.get().taskMemoryManager()` in its constructor and uses the stored reference thereafter. **Memory pool operations call into the JVM.** `CometUnifiedMemoryPool` and `CometFairMemoryPool` call `acquireMemory()` / `releaseMemory()` via JNI whenever DataFusion operators grow or shrink memory reservations. This happens on whatever thread the operator is executing on. These calls are thread-safe (they use stored `GlobalRef`s, not thread-locals), but they do trigger `AttachCurrentThread`. **Scalar subqueries call into the JVM.** `Subquery::evaluate()` calls static methods on `CometScalarSubquery` via JNI. These use a static `HashMap`, not thread-locals, so they are safe from any thread. **Parquet encryption calls into the JVM.** `CometKeyRetriever::retrieve_key()` calls the JVM to unwrap decryption keys during Parquet reads. It uses a stored `GlobalRef` and a cached `JMethodID`, so it is safe from any thread. ### The tokio runtime The runtime is created once per executor JVM in a `Lazy` static: - **Worker threads:** `num_cpus` by default, configurable via `COMET_WORKER_THREADS` - **Max blocking threads:** 512 by default, configurable via `COMET_MAX_BLOCKING_THREADS` - All async I/O (S3, HTTP, Parquet reads) runs on worker threads as non-blocking futures ### Summary of what is safe and what is not | Pattern | Safe? | Notes | | ----------------------------------------- | ------ | ---------------------------------------- | | `Arc` shared across operators | Yes | Standard Rust thread safety | | `JVMClasses::get_env()` from tokio worker | Yes | Attaches thread to JVM automatically | | `thread_local!` in operator code | **No** | Tokio moves tasks between threads | | `TaskContext.get()` in JVM callback | **No** | Returns `null` on non-executor threads | | Storing `JNIEnv` in an operator | **No** | `JNIEnv` is thread-specific | | Capturing state at plan creation time | Yes | Runs on executor thread, store in struct | ## Global singletons Comet code runs in both the driver and executor JVM processes, and different parts of the codebase run in each. Global singletons have **process lifetime** — they are created once and never dropped until the JVM exits. Since multiple Spark jobs, queries, and tasks share the same process, this makes it difficult to reason about what state a singleton holds and whether it is still valid. ### How to recognize them **Rust:** `static` variables using `OnceLock`, `LazyLock`, `OnceCell`, `Lazy`, or `lazy_static!`: ```rust static TOKIO_RUNTIME: OnceLock = OnceLock::new(); static TASK_SHARED_MEMORY_POOLS: Lazy>> = Lazy::new(..); ``` **Java:** `static` fields, especially mutable collections: ```java private static final HashMap> subqueryMap = new HashMap<>(); ``` **Scala:** `object` declarations (companion objects are JVM singletons) holding mutable state: ```scala object MyCache { private val cache = new ConcurrentHashMap[String, Value]() } ``` ### Why they are dangerous - **Credential staleness.** A singleton caching an authenticated client will hold stale credentials after token rotation, causing silent failures mid-job. - **Unbounded growth.** A cache keyed by file path or configuration grows with every query but never shrinks. Over hours of process uptime this becomes a memory leak. - **Cross-job contamination.** Different Spark jobs on the same process may use different configurations. A singleton initialized by the first job silently serves wrong state to subsequent jobs. - **Testing difficulty.** Global state persists across test cases, making tests order-dependent. ### When a singleton is acceptable Some state genuinely has process lifetime: | Singleton | Why it is safe | | --------------------------------------------- | --------------------------------------------------- | | `TOKIO_RUNTIME` | One runtime per executor, no configuration variance | | `JAVA_VM` / `JVM_CLASSES` | One JVM per process, set once at JNI load | | `OperatorRegistry` / `ExpressionRegistry` | Immutable after initialization | | Compiled `Regex` patterns (`LazyLock`) | Stateless and immutable | ### When to avoid a singleton If any of these apply, do **not** use a global singleton: - The state depends on configuration that can vary between jobs or queries - The state holds credentials or authenticated connections that will not expire or invalidate appropriately - The state grows proportionally to the number of queries or files processed - The state needs cleanup or refresh during process lifetime Instead, scope state to the plan or task by adding the cache as a field in an existing session or context object. If a singleton is truly needed, add a comment explaining why `static` is the right lifetime, whether the cache is bounded, and how credential refresh is handled (if applicable). ## Development Setup 1. Make sure `JAVA_HOME` is set and point to JDK using [support matrix](../user-guide/latest/installation.md) 2. Install Rust toolchain. The easiest way is to use [rustup](https://rustup.rs). ## Build & Test A few common commands are specified in project's `Makefile`: - `make`: compile the entire project, but don't run tests - `make test-rust`: compile the project and run tests in Rust side - `make test-jvm`: compile the project and run tests in Java side - `make test`: compile the project and run tests in both Rust and Java side. - `make release`: compile the project and creates a release build. This is useful when you want to test Comet local installation in another project such as Spark. - `make clean`: clean up the workspace ## Common Build and Test Pitfalls ### Native Code Must Be Built First The native Rust code must be compiled before running JVM tests. If you skip this step, tests will fail because they cannot find the native library. Always run `make core` (or `cd native && cargo build`) before running Maven tests. ```sh # Correct order make core # Build native code first ./mvnw test -Dsuites="..." # Then run JVM tests ``` ### Debug vs Release Mode There is no need to use release mode (`make release`) during normal development. Debug builds are faster to compile and provide better error messages. Only use release mode when: - Running benchmarks - Testing performance - Creating a release build for deployment For regular development and testing, use `make` or `make core` which build in debug mode. ### Running Rust Tests When running Rust tests directly with `cargo test`, the JVM library (`libjvm.so`) must be on your library path. Set the `LD_LIBRARY_PATH` environment variable to include your JDK's `lib/server` directory: ```sh # Find your libjvm.so location (example for typical JDK installation) export LD_LIBRARY_PATH=$JAVA_HOME/lib/server:$LD_LIBRARY_PATH # Now you can run Rust tests cd native && cargo test ``` Alternatively, use `make test-rust` which handles the JVM compilation dependency automatically. ### Avoid Using `-pl` to Select Modules When running Maven tests, avoid using `-pl spark` to select only the spark module. This can cause Maven to pick up the `common` module from your local Maven repository instead of using the current codebase, leading to inconsistent test results: ```sh # Avoid this - may use stale common module from local repo ./mvnw test -pl spark -Dsuites="..." # Do this instead - builds and tests with current code ./mvnw test -Dsuites="..." ``` ### Use `wildcardSuites` for Running Tests When running specific test suites, use `wildcardSuites` instead of `suites` for more flexible matching. The `wildcardSuites` parameter allows partial matching of suite names: ```sh # Run all suites containing "CometCast" ./mvnw test -DwildcardSuites="CometCast" # Run specific suite with filter ./mvnw test -Dsuites="org.apache.comet.CometCastSuite valid" ``` ## Development Environment Comet is a multi-language project with native code written in Rust and JVM code written in Java and Scala. For Rust code, the CLion IDE is recommended. For JVM code, IntelliJ IDEA is recommended. Before opening the project in an IDE, make sure to run `make` first to generate the necessary files for the IDEs. Currently, it's mostly about generating protobuf message classes for the JVM side. It's only required to run `make` once after cloning the repo. ### IntelliJ IDEA First make sure to install the Scala plugin in IntelliJ IDEA. After that, you can open the project in IntelliJ IDEA. The IDE should automatically detect the project structure and import as a Maven project. Comet uses generated source files that are too large for IntelliJ's default size limit for code inspections. To avoid IDE errors (missing definitions, etc.) caused by IntelliJ skipping these generated files, modify [IntelliJ's Platform Properties](https://intellij-support.jetbrains.com/hc/en-us/articles/206544869-Configuring-JVM-options-and-platform-properties) by going to `Help -> Edit Custom Properties...`. For example, adding `idea.max.intellisense.filesize=16384` increases the file size limit to 16 MB. ### CLion First make sure to install the Rust plugin in CLion or you can use the dedicated Rust IDE: RustRover. After that you can open the project in CLion. The IDE should automatically detect the project structure and import as a Cargo project. ### SQL file tests (recommended for expressions) For testing expressions and operators, prefer using SQL file tests over writing Scala test code. SQL file tests are plain `.sql` files that are automatically discovered and executed -- no Scala code to write, and no recompilation needed when tests change. This makes it easy to iterate quickly and to get good coverage of edge cases and argument combinations. See the [SQL File Tests](sql-file-tests) guide for the full documentation on how to write and run these tests. ### Running Tests in IDEA Like other Maven projects, you can run tests in IntelliJ IDEA by right-clicking on the test class or test method and selecting "Run" or "Debug". However if the tests is related to the native side. Please make sure to run `make core` or `cd native && cargo build` before running the tests in IDEA. ### Running Tests from command line It is possible to specify which ScalaTest suites you want to run from the CLI using the `suites` argument, for example if you only want to execute the test cases that contains _valid_ in their name in `org.apache.comet.CometCastSuite` you can use ```sh ./mvnw test -Dtest=none -Dsuites="org.apache.comet.CometCastSuite valid" ``` Other options for selecting specific suites are described in the [ScalaTest Maven Plugin documentation](https://www.scalatest.org/user_guide/using_the_scalatest_maven_plugin) ## Plan Stability Testing Comet has a plan stability testing framework that can be used to test the stability of the query plans generated by Comet. The plan stability testing framework is located in the `spark` module. ### Using the Helper Script The easiest way to regenerate golden files is to use the provided script: ```sh # Regenerate golden files for all Spark versions ./dev/regenerate-golden-files.sh # Regenerate only for a specific Spark version ./dev/regenerate-golden-files.sh --spark-version 3.5 ``` The script verifies that JDK 17+ is configured (required for Spark 4.0), installs Comet for each Spark version, and runs the plan stability tests with `SPARK_GENERATE_GOLDEN_FILES=1`. ### Manual Instructions Alternatively, you can run the tests manually using the following commands. Note that the output files get written to `$SPARK_HOME`. The tests can be run with: ```sh export SPARK_HOME=`pwd` ./mvnw -Dsuites="org.apache.spark.sql.comet.CometTPCDSV1_4_PlanStabilitySuite" -Pspark-3.4 -nsu test ./mvnw -Dsuites="org.apache.spark.sql.comet.CometTPCDSV1_4_PlanStabilitySuite" -Pspark-3.5 -nsu test ./mvnw -Dsuites="org.apache.spark.sql.comet.CometTPCDSV1_4_PlanStabilitySuite" -Pspark-4.0 -nsu test ``` and ```sh export SPARK_HOME=`pwd` ./mvnw -Dsuites="org.apache.spark.sql.comet.CometTPCDSV2_7_PlanStabilitySuite" -Pspark-3.4 -nsu test ./mvnw -Dsuites="org.apache.spark.sql.comet.CometTPCDSV2_7_PlanStabilitySuite" -Pspark-3.5 -nsu test ./mvnw -Dsuites="org.apache.spark.sql.comet.CometTPCDSV2_7_PlanStabilitySuite" -Pspark-4.0 -nsu test ``` If your pull request changes the query plans generated by Comet, you should regenerate the golden files. To regenerate the golden files, you can run the following commands. ```sh export SPARK_HOME=`pwd` SPARK_GENERATE_GOLDEN_FILES=1 ./mvnw -Dsuites="org.apache.spark.sql.comet.CometTPCDSV1_4_PlanStabilitySuite" -Pspark-3.4 -nsu test SPARK_GENERATE_GOLDEN_FILES=1 ./mvnw -Dsuites="org.apache.spark.sql.comet.CometTPCDSV1_4_PlanStabilitySuite" -Pspark-3.5 -nsu test SPARK_GENERATE_GOLDEN_FILES=1 ./mvnw -Dsuites="org.apache.spark.sql.comet.CometTPCDSV1_4_PlanStabilitySuite" -Pspark-4.0 -nsu test ``` and ```sh export SPARK_HOME=`pwd` SPARK_GENERATE_GOLDEN_FILES=1 ./mvnw -Dsuites="org.apache.spark.sql.comet.CometTPCDSV2_7_PlanStabilitySuite" -Pspark-3.4 -nsu test SPARK_GENERATE_GOLDEN_FILES=1 ./mvnw -Dsuites="org.apache.spark.sql.comet.CometTPCDSV2_7_PlanStabilitySuite" -Pspark-3.5 -nsu test SPARK_GENERATE_GOLDEN_FILES=1 ./mvnw -Dsuites="org.apache.spark.sql.comet.CometTPCDSV2_7_PlanStabilitySuite" -Pspark-4.0 -nsu test ``` ## Benchmark There's a `make` command to run micro benchmarks in the repo. For instance: ``` make benchmark-org.apache.spark.sql.benchmark.CometReadBenchmark ``` To run TPC-H or TPC-DS micro benchmarks, please follow the instructions in the respective source code, e.g., `CometTPCHQueryBenchmark`. ## Debugging Comet is a multi-language project with native code written in Rust and JVM code written in Java and Scala. It is possible to debug both native and JVM code concurrently as described in the [DEBUGGING guide](debugging) ## Submitting a Pull Request Before submitting a pull request, follow this checklist to ensure your changes are ready: ### 1. Format Your Code Comet uses `cargo fmt`, [Scalafix](https://github.com/scalacenter/scalafix) and [Spotless](https://github.com/diffplug/spotless/tree/main/plugin-maven) to automatically format the code. Run the following command to format all code: ```sh make format ``` ### 2. Build and Verify After formatting, run a full build to ensure everything compiles correctly and generated documentation is up to date: ```sh make ``` This builds both native and JVM code. Fix any compilation errors before proceeding. ### 3. Run Clippy (Recommended) It's strongly recommended to run Clippy locally to catch potential issues before the CI/CD pipeline does. You can run the same Clippy checks used in CI/CD with: ```bash cd native cargo clippy --color=never --all-targets --workspace -- -D warnings ``` Make sure to resolve any Clippy warnings before submitting your pull request, as the CI/CD pipeline will fail if warnings are present. ### 4. Run Tests Run the relevant tests for your changes: ```sh # Run all tests make test # Or run only Rust tests make test-rust # Or run only JVM tests (native must be built first) make test-jvm ``` ### 5. Register New Test Suites in CI Comet's CI does not automatically discover test suites. Instead, test suites are explicitly listed in the GitHub Actions workflow files so they can be grouped by category and run as separate parallel jobs. This reduces overall CI time. If you add a new Scala test suite, you must add it to the `suite` matrix in **both** workflow files: - `.github/workflows/pr_build_linux.yml` - `.github/workflows/pr_build_macos.yml` Each file contains a `suite` matrix with named groups such as `fuzz`, `shuffle`, `parquet`, `csv`, `exec`, `expressions`, and `sql`. Add your new suite's fully qualified class name to the appropriate group. For example, if you add a new expression test suite, add it to the `expressions` group: ```yaml - name: "expressions" value: | org.apache.comet.CometExpressionSuite # ... existing suites ... org.apache.comet.YourNewExpressionSuite # <-- add here ``` Choose the group that best matches the area your test covers: | Group | Covers | | ------------- | ---------------------------------------------------------- | | `fuzz` | Fuzz testing and data generation | | `shuffle` | Shuffle operators and related exchange behavior | | `parquet` | Parquet read/write and native reader tests | | `csv` | CSV native read tests | | `exec` | Execution operators, joins, aggregates, plan rules, TPC-\* | | `expressions` | Expression evaluation, casts, and SQL file tests | | `sql` | SQL-level behavior tests | **Important:** The suite lists in both workflow files must stay in sync. A separate CI check (`.github/workflows/pr_missing_suites.yml`) runs `dev/ci/check-suites.py` on every pull request. It scans for all `*Suite.scala` files in the repository and verifies that each one appears in both workflow files. If any suite is missing, this check will fail and block the PR. ### Pre-PR Summary ```sh make format # Format code make # Build everything and update generated docs make test # Run tests (optional but recommended) ``` ## How to format `.md` document We are using `prettier` to format `.md` files. You can either use `npm i -g prettier` to install it globally or use `npx` to run it as a standalone binary. Using `npx` required a working node environment. Upgrading to the latest prettier is recommended (by adding `--upgrade` to the `npm` command). ```bash $ prettier --version 2.3.0 ``` After you've confirmed your prettier version, you can format all the `.md` files: ```bash npx prettier "**/*.md" --write ``` ================================================ FILE: docs/source/contributor-guide/expression-audit-log.md ================================================ # Expression Audit Log This document tracks which Comet expressions have been audited against their Spark implementations for correctness and test coverage. Each audit compares the Comet implementation against the Spark source code for the listed versions, reviews existing test coverage, identifies gaps, and adds missing tests where needed. ## Audited Expressions | Expression | Spark Versions Checked | Date | Findings | | -------------- | ---------------------- | ---------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `array_insert` | 3.4.3, 3.5.8, 4.0.1 | 2026-04-02 | No behavioral differences across Spark versions. Fixed `nullable()` metadata bug (did not account for `pos_expr`). Added SQL tests for multiple types (string, boolean, double, float, long, short, byte), literal arguments, null handling, negative indices, out-of-bounds padding, special float values (NaN, Infinity), multibyte UTF-8, and legacy negative index mode. Known incompatibility: pos=0 error message differs from Spark's `INVALID_INDEX_OF_ZERO`. | ================================================ FILE: docs/source/contributor-guide/ffi.md ================================================ # Arrow FFI Usage in Comet ## Overview Comet uses the [Arrow C Data Interface](https://arrow.apache.org/docs/format/CDataInterface.html) for zero-copy data transfer in two directions: 1. **JVM → Native**: Native code pulls batches from JVM using `CometBatchIterator` 2. **Native → JVM**: JVM pulls batches from native code using `CometExecIterator` The following diagram shows an example of the end-to-end flow for a query stage. ![Diagram of Comet Data Flow](/_static/images/comet-dataflow.svg) Both scenarios use the same FFI mechanism but have different ownership semantics and memory management implications. ## Arrow FFI Basics The Arrow C Data Interface defines two C structures: - `ArrowArray`: Contains pointers to data buffers and metadata - `ArrowSchema`: Contains type information ### Key Characteristics - **Zero-copy**: Data buffers can be shared across language boundaries without copying - **Ownership transfer**: Clear semantics for who owns and must free the data - **Release callbacks**: Custom cleanup functions for proper resource management ## JVM → Native Data Flow (ScanExec) ### Architecture When native code needs data from the JVM, it uses `ScanExec` which calls into `CometBatchIterator`: ``` ┌─────────────────┐ │ Spark/Scala │ │ CometExecIter │ └────────┬────────┘ │ produces batches ▼ ┌─────────────────┐ │ CometBatchIter │ ◄─── JNI call from native │ (JVM side) │ └────────┬────────┘ │ Arrow FFI │ (transfers ArrowArray/ArrowSchema pointers) ▼ ┌─────────────────┐ │ ScanExec │ │ (Rust/native) │ └────────┬────────┘ │ ▼ ┌─────────────────┐ │ DataFusion │ │ operators │ └─────────────────┘ ``` ### FFI Transfer Process The data transfer happens in `ScanExec::get_next()`: ```rust // 1. Allocate FFI structures on native side (Rust heap) for _ in 0..num_cols { let arrow_array = Rc::new(FFI_ArrowArray::empty()); let arrow_schema = Rc::new(FFI_ArrowSchema::empty()); let array_ptr = Rc::into_raw(arrow_array) as i64; let schema_ptr = Rc::into_raw(arrow_schema) as i64; // Store pointers... } // 2. Call JVM to populate FFI structures let num_rows: i32 = unsafe { jni_call!(env, comet_batch_iterator(iter).next(array_obj, schema_obj) -> i32)? }; // 3. Import data from FFI structures for i in 0..num_cols { let array_data = ArrayData::from_spark((array_ptr, schema_ptr))?; let array = make_array(array_data); // ... process array } ``` ### Memory Layout When a batch is transferred from JVM to native: ``` JVM Heap: Native Memory: ┌──────────────────┐ ┌──────────────────┐ │ ColumnarBatch │ │ FFI_ArrowArray │ │ ┌──────────────┐ │ │ ┌──────────────┐ │ │ │ ArrowBuf │─┼──────────────>│ │ buffers[0] │ │ │ │ (off-heap) │ │ │ │ (pointer) │ │ │ └──────────────┘ │ │ └──────────────┘ │ └──────────────────┘ └──────────────────┘ │ │ │ │ Off-heap Memory: │ ┌──────────────────┐ <──────────────────────┘ │ Actual Data │ │ (e.g., int32[]) │ └──────────────────┘ ``` **Key Point**: The actual data buffers can be off-heap, but the `ArrowArray` and `ArrowSchema` wrapper objects are **always allocated on the JVM heap**. ### Wrapper Object Lifecycle When arrays are created in the JVM and passed to native code, the JVM creates the array data off-heap and creates wrapper objects `ArrowArray` and `ArrowSchema` on-heap. These wrapper objects can consume significant memory over time. ``` Per batch overhead on JVM heap: - ArrowArray object: ~100 bytes - ArrowSchema object: ~100 bytes - Per column: ~200 bytes - 100 columns × 1000 batches = ~20 MB of wrapper objects ``` When native code pulls batches from the JVM, the JVM wrapper objects are kept alive until the native code drops all references to the arrays. When operators such as `SortExec` fetch many batches and buffer them in native code, the number of wrapper objects in Java on-heap memory keeps growing until the batches are released in native code at the end of the sort operation. ### Ownership Transfer The Arrow C data interface supports ownership transfer by registering callbacks in the C struct that is passed over the JNI boundary for the function to delete the array data. For example, the `ArrowArray` struct has: ```c // Release callback void (*release)(struct ArrowArray*); ``` Comet currently does not always follow best practice around ownership transfer because there are some cases where Comet JVM code will retain references to arrays after passing them to native code and may mutate the underlying buffers. There is an `arrow_ffi_safe` flag in the protocol buffer definition of `Scan` that indicates whether ownership is being transferred according to the Arrow C data interface specification. ```protobuf message Scan { repeated spark.spark_expression.DataType fields = 1; // The source of the scan (e.g. file scan, broadcast exchange, shuffle, etc). This // is purely for informational purposes when viewing native query plans in // debug mode. string source = 2; // Whether native code can assume ownership of batches that it receives bool arrow_ffi_safe = 3; } ``` #### When ownership is NOT transferred to native: If the data originates from a scan that uses mutable buffers then ownership is not transferred to native and the JVM may re-use the underlying buffers in the future. It is critical that the native code performs a deep copy of the arrays if the arrays are to be buffered by operators such as `SortExec` or `ShuffleWriterExec`, otherwise data corruption is likely to occur. #### When ownership IS transferred to native: When ownership is transferred, it is safe to buffer batches in native. However, JVM wrapper objects will not be released until the native batches are dropped. This can lead to OOM or GC pressure if there is not enough Java heap memory configured. ## Native → JVM Data Flow (CometExecIterator) ### Architecture When JVM needs results from native execution: ``` ┌─────────────────┐ │ DataFusion Plan │ │ (native) │ └────────┬────────┘ │ produces RecordBatch ▼ ┌─────────────────┐ │ CometExecIter │ │ (Rust/native) │ └────────┬────────┘ │ Arrow FFI │ (transfers ArrowArray/ArrowSchema pointers) ▼ ┌─────────────────┐ │ CometExecIter │ ◄─── JNI call from Spark │ (Scala side) │ └────────┬────────┘ │ ▼ ┌─────────────────┐ │ Spark Actions │ │ (collect, etc) │ └─────────────────┘ ``` ### FFI Transfer Process The transfer happens in `CometExecIterator::getNextBatch()`: ```scala // Scala side def getNextBatch(): ColumnarBatch = { val batchHandle = Native.getNextBatch(nativeHandle) // Import from FFI structures val vectors = (0 until schema.length).map { i => val array = Array.empty[Long](1) val schemaPtr = Array.empty[Long](1) // Get FFI pointers from native Native.exportVector(batchHandle, i, array, schemaPtr) // Import into Arrow Java Data.importVector(allocator, array(0), schemaPtr(0)) } new ColumnarBatch(vectors.toArray, numRows) } ``` ```rust // Native side (simplified) #[no_mangle] pub extern "system" fn Java_..._getNextBatch( env: JNIEnv, handle: jlong, ) -> jlong { let context = get_exec_context(handle)?; let batch = context.stream.next().await?; // Store batch and return handle let batch_handle = Box::into_raw(Box::new(batch)) as i64; batch_handle } #[no_mangle] pub extern "system" fn Java_..._exportVector( env: JNIEnv, batch_handle: jlong, col_idx: jint, array_ptr: jlongArray, schema_ptr: jlongArray, ) { let batch = get_batch(batch_handle)?; let array = batch.column(col_idx); // Export to FFI structures let (array_ffi, schema_ffi) = to_ffi(array.to_data())?; // Write pointers back to JVM env.set_long_array_region(array_ptr, 0, &[array_ffi as i64])?; env.set_long_array_region(schema_ptr, 0, &[schema_ffi as i64])?; } ``` ### Wrapper Object Lifecycle (Native → JVM) ``` Time Native Memory JVM Heap Off-heap/Native ──────────────────────────────────────────────────────────────────────── t0 RecordBatch produced - Data in native in DataFusion t1 FFI_ArrowArray created - Data in native FFI_ArrowSchema created (native heap) t2 Pointers exported to JVM ArrowBuf created Data in native (wraps native ptr) t3 FFI structures kept alive Spark processes Data in native via batch handle ColumnarBatch ✓ Valid t4 Batch handle released ArrowBuf freed Data freed Release callback runs (triggers native (via release release callback) callback) ``` **Key Difference from JVM → Native**: - Native code controls lifecycle through batch handle - JVM creates `ArrowBuf` wrappers that point to native memory - Release callback ensures proper cleanup when JVM is done - No GC pressure issue because native allocator manages the data ### Release Callbacks Critical for proper cleanup: ```rust // Native release callback (simplified) extern "C" fn release_batch(array: *mut FFI_ArrowArray) { if !array.is_null() { unsafe { // Free the data buffers for buffer in (*array).buffers { drop(Box::from_raw(buffer)); } // Free the array structure itself drop(Box::from_raw(array)); } } } ``` When JVM is done with the data: ```java // ArrowBuf.close() triggers the release callback arrowBuf.close(); // → calls native release_batch() ``` ## Memory Ownership Rules ### JVM → Native | Scenario | `arrow_ffi_safe` | Ownership | Action Required | | ------------------ | ---------------- | ----------- | -------------------------------------- | | Temporary scan | `false` | JVM keeps | **Must deep copy** to avoid corruption | | Ownership transfer | `true` | Native owns | Copy only to unpack dictionaries | ### Native → JVM | Scenario | Ownership | Action Required | | --------- | -------------------------------- | ---------------------------------------------------------- | | All cases | Native allocates, JVM references | JVM must call `close()` to trigger native release callback | ## Further Reading - [Arrow C Data Interface Specification](https://arrow.apache.org/docs/format/CDataInterface.html) - [Arrow Java FFI Implementation](https://github.com/apache/arrow/tree/main/java/c) - [Arrow Rust FFI Implementation](https://docs.rs/arrow/latest/arrow/ffi/) ================================================ FILE: docs/source/contributor-guide/iceberg-spark-tests.md ================================================ # Running Iceberg Spark Tests Running Apache Iceberg's Spark tests with Comet enabled is a good way to ensure that Comet produces the same results as Spark when reading Iceberg tables. To enable this, we apply diff files to the Apache Iceberg source code so that Comet is loaded when we run the tests. Here is an overview of the changes that the diffs make to Iceberg: - Configure Comet as a dependency and set the correct version in `libs.versions.toml` and `build.gradle` - Delete upstream Comet reader classes that reference legacy Comet APIs removed in [#3739]. These classes were added upstream in [apache/iceberg#15674] and depend on Comet's old Iceberg Java integration. Since Comet now uses a native Iceberg scan, these classes fail to compile and must be removed. - Configure test base classes (`TestBase`, `ExtensionsTestBase`, `ScanTestBase`, etc.) to load the Comet Spark plugin and shuffle manager [#3739]: https://github.com/apache/datafusion-comet/pull/3739 [apache/iceberg#15674]: https://github.com/apache/iceberg/pull/15674 ## 1. Install Comet Run `make release` in Comet to install the Comet JAR into the local Maven repository, specifying the Spark version. ```shell PROFILES="-Pspark-3.5" make release ``` ## 2. Clone Iceberg and Apply Diff Clone Apache Iceberg locally and apply the diff file from Comet against the matching tag. ```shell git clone git@github.com:apache/iceberg.git apache-iceberg cd apache-iceberg git checkout apache-iceberg-1.8.1 git apply ../datafusion-comet/dev/diffs/iceberg/1.8.1.diff ``` ## 3. Run Iceberg Spark Tests ```shell ENABLE_COMET=true ./gradlew -DsparkVersions=3.5 -DscalaVersion=2.13 -DflinkVersions= -DkafkaVersions= \ :iceberg-spark:iceberg-spark-3.5_2.13:test \ -Pquick=true -x javadoc ``` The three Gradle targets tested in CI are: | Gradle Target | What It Covers | | --------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `iceberg-spark-:test` | Core read/write paths (Parquet, Avro, ORC, vectorized), scan operations, filtering, bloom filters, runtime filtering, deletion handling, structured streaming, DDL/DML (create/alter/drop, writes, deletes), filter and aggregate pushdown, actions (snapshot expiration, file rewriting, orphan cleanup, table migration), serialization, and data format conversions. | | `iceberg-spark-extensions-:test` | SQL extensions: stored procedures (migrate, snapshot, cherrypick, rollback, rewrite-data-files, rewrite-manifests, expire-snapshots, remove-orphan-files, etc.), row-level operations (copy-on-write and merge-on-read update/delete/merge), DDL extensions (branches, tags, alter schema, partition fields), changelog tables/views, metadata tables, and views. | | `iceberg-spark-runtime-:integrationTest` | A single smoke test (`SmokeTest.java`) that validates the shaded runtime JAR. The `spark-runtime` module has no main source — it packages Iceberg and all dependencies into a shaded uber-JAR. The smoke test exercises basic create, insert, merge, query, partition field, and sort order operations to confirm the shaded JAR works end-to-end. | ## Updating Diffs To update a diff (e.g. after modifying test configuration), apply the existing diff, make changes, then regenerate: ```shell cd apache-iceberg git reset --hard apache-iceberg-1.8.1 && git clean -fd git apply ../datafusion-comet/dev/diffs/iceberg/1.8.1.diff # Make changes, then run spotless to fix formatting ./gradlew spotlessApply # Stage any new or deleted files, then generate the diff git add -A git diff apache-iceberg-1.8.1 > ../datafusion-comet/dev/diffs/iceberg/1.8.1.diff ``` Repeat for each Iceberg version (1.8.1, 1.9.1, 1.10.0). The file contents differ between versions, so each diff must be generated against its own tag. ## Running Tests in CI The `iceberg_spark_test.yml` workflow applies these diffs and runs the three Gradle targets above against each Iceberg version. The test matrix covers Spark 3.4 and 3.5 across Iceberg 1.8.1, 1.9.1, and 1.10.0 with Java 11 and 17. The workflow runs on all pull requests and pushes to the main branch. ================================================ FILE: docs/source/contributor-guide/index.md ================================================ # Comet Contributor Guide ```{toctree} :maxdepth: 2 :caption: Contributor Guide Getting Started Comet Plugin Overview Arrow FFI JVM Shuffle Native Shuffle Parquet Scans Development Guide Debugging Guide Benchmarking Guide Adding a New Operator Adding a New Expression Tracing Profiling Spark SQL Tests Iceberg Spark Tests SQL File Tests Bug Triage Roadmap Release Process Github and Issue Tracker ``` ================================================ FILE: docs/source/contributor-guide/jvm_shuffle.md ================================================ # JVM Shuffle This document describes Comet's JVM-based columnar shuffle implementation (`CometColumnarShuffle`), which writes shuffle data in Arrow IPC format using JVM code with native encoding. For the fully native alternative, see [Native Shuffle](native_shuffle.md). ## Overview Comet provides two shuffle implementations: - **CometNativeShuffle** (`CometExchange`): Fully native shuffle using Rust. Takes columnar input directly from Comet native operators and performs partitioning in native code. - **CometColumnarShuffle** (`CometColumnarExchange`): JVM-based shuffle that operates on rows internally, buffers `UnsafeRow`s in memory pages, and uses native code (via JNI) to encode them to Arrow IPC format. Uses Spark's partitioner for partition assignment. Can accept either row-based or columnar input (columnar input is converted to rows via `ColumnarToRowExec`). The JVM shuffle is selected via `CometShuffleDependency.shuffleType`. ## When JVM Shuffle is Used JVM shuffle (`CometColumnarExchange`) is used instead of native shuffle (`CometExchange`) in the following cases: 1. **Shuffle mode is explicitly set to "jvm"**: When `spark.comet.exec.shuffle.mode` is set to `jvm`. 2. **Child plan is not a Comet native operator**: When the child plan is a Spark row-based operator (not a `CometPlan`), JVM shuffle is the only option since native shuffle requires columnar input from Comet operators. 3. **Unsupported partition key types**: For `HashPartitioning` and `RangePartitioning`, native shuffle only supports primitive types as partition keys. Complex types (struct, array, map) cannot be used as partition keys in native shuffle and will fall back to JVM columnar shuffle. Note that complex types are fully supported as data columns in both implementations. ## Input Handling ### Spark Row-Based Input When the child plan is a Spark row-based operator, `CometColumnarExchange` calls `child.execute()` which returns an `RDD[InternalRow]`. The rows flow directly to the JVM shuffle writers. ### Comet Columnar Input When the child plan is a Comet native operator (e.g., `CometHashAggregate`) but JVM shuffle is selected (due to shuffle mode setting or unsupported partitioning), `CometColumnarExchange` still calls `child.execute()`. Comet operators implement `doExecute()` by wrapping themselves with `ColumnarToRowExec`: ```scala // In CometExec base class override def doExecute(): RDD[InternalRow] = ColumnarToRowExec(this).doExecute() ``` This means the data path becomes: ``` Comet Native (columnar) → ColumnarToRowExec → rows → JVM Shuffle → Arrow IPC → columnar ``` This is less efficient than native shuffle which avoids the columnar-to-row conversion: ``` Comet Native (columnar) → Native Shuffle → Arrow IPC → columnar ``` ### Why Use Spark's Partitioner? JVM shuffle uses row-based input so it can leverage Spark's existing partitioner infrastructure (`partitioner.getPartition(key)`). This allows Comet to support all of Spark's partitioning schemes without reimplementing them in Rust. Native shuffle, by contrast, serializes the partitioning scheme to protobuf and implements the partitioning logic in native code. ## Architecture ``` ┌─────────────────────────────────────────────────────────────────────────┐ │ CometShuffleManager │ │ - Extends Spark's ShuffleManager │ │ - Routes to appropriate writer/reader based on ShuffleHandle type │ └─────────────────────────────────────────────────────────────────────────┘ │ ┌────────────────────────┼────────────────────────┐ ▼ ▼ ▼ ┌─────────────────────┐ ┌─────────────────────┐ ┌─────────────────────┐ │ CometBypassMerge- │ │ CometUnsafe- │ │ CometNative- │ │ SortShuffleWriter │ │ ShuffleWriter │ │ ShuffleWriter │ │ (hash-based) │ │ (sort-based) │ │ (fully native) │ └─────────────────────┘ └─────────────────────┘ └─────────────────────┘ │ │ ▼ ▼ ┌─────────────────────┐ ┌─────────────────────┐ │ CometDiskBlock- │ │ CometShuffleExternal│ │ Writer │ │ Sorter │ └─────────────────────┘ └─────────────────────┘ │ │ └────────────┬───────────┘ ▼ ┌─────────────────────┐ │ SpillWriter │ │ (native encoding │ │ via JNI) │ └─────────────────────┘ ``` ## Key Classes ### Shuffle Manager | Class | Location | Description | | ------------------------ | ------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------ | | `CometShuffleManager` | `.../shuffle/CometShuffleManager.scala` | Entry point. Extends Spark's `ShuffleManager`. Selects writer/reader based on handle type. Delegates non-Comet shuffles to `SortShuffleManager`. | | `CometShuffleDependency` | `.../shuffle/CometShuffleDependency.scala` | Extends `ShuffleDependency`. Contains `shuffleType` (`CometColumnarShuffle` or `CometNativeShuffle`) and schema info. | ### Shuffle Handles | Handle | Writer Strategy | | ----------------------------------- | --------------------------------------------------------- | | `CometBypassMergeSortShuffleHandle` | Hash-based: one file per partition, merged at end | | `CometSerializedShuffleHandle` | Sort-based: records sorted by partition ID, single output | | `CometNativeShuffleHandle` | Fully native shuffle | Selection logic in `CometShuffleManager.shouldBypassMergeSort()`: - Uses bypass if partitions < threshold AND partitions × cores ≤ max threads - Otherwise uses sort-based to avoid OOM from many concurrent writers ### Writers | Class | Location | Description | | ----------------------------------- | ---------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------- | | `CometBypassMergeSortShuffleWriter` | `.../shuffle/CometBypassMergeSortShuffleWriter.java` | Hash-based writer. Creates one `CometDiskBlockWriter` per partition. Supports async writes. | | `CometUnsafeShuffleWriter` | `.../shuffle/CometUnsafeShuffleWriter.java` | Sort-based writer. Uses `CometShuffleExternalSorter` to buffer and sort records, then merges spill files. | | `CometDiskBlockWriter` | `.../shuffle/CometDiskBlockWriter.java` | Buffers rows in memory pages for a single partition. Spills to disk via native encoding. Used by bypass writer. | | `CometShuffleExternalSorter` | `.../shuffle/sort/CometShuffleExternalSorter.java` | Buffers records across all partitions, sorts by partition ID, spills sorted data. Used by unsafe writer. | | `SpillWriter` | `.../shuffle/SpillWriter.java` | Base class for spill logic. Manages memory pages and calls `Native.writeSortedFileNative()` for Arrow IPC encoding. | ### Reader | Class | Location | Description | | ------------------------------ | ------------------------------------------------ | -------------------------------------------------------------------------------------------------- | | `CometBlockStoreShuffleReader` | `.../shuffle/CometBlockStoreShuffleReader.scala` | Fetches shuffle blocks via `ShuffleBlockFetcherIterator`. Decodes Arrow IPC to `ColumnarBatch`. | | `NativeBatchDecoderIterator` | `.../shuffle/NativeBatchDecoderIterator.scala` | Reads compressed Arrow IPC batches from input stream. Calls `Native.decodeShuffleBlock()` via JNI. | ## Data Flow ### Write Path 1. `ShuffleWriteProcessor` calls `CometShuffleManager.getWriter()` 2. Writer receives `Iterator[Product2[K, V]]` where V is `UnsafeRow` 3. Rows are serialized and buffered in off-heap memory pages 4. When memory threshold or batch size is reached, `SpillWriter.doSpilling()` is called 5. Native code (`Native.writeSortedFileNative()`) converts rows to Arrow arrays and writes IPC format 6. For bypass writer: partition files are concatenated into final output 7. For sort writer: spill files are merged ### Read Path 1. `CometBlockStoreShuffleReader.read()` creates `ShuffleBlockFetcherIterator` 2. For each block, `NativeBatchDecoderIterator` reads the IPC stream 3. Native code (`Native.decodeShuffleBlock()`) decompresses and decodes to Arrow arrays 4. Arrow FFI imports arrays as `ColumnarBatch` ## Memory Management - `CometShuffleMemoryAllocator`: Custom allocator for off-heap memory pages - Memory is allocated in pages; when allocation fails, writers spill to disk - `CometDiskBlockWriter` coordinates spilling across all partition writers (largest first) - Async spilling is supported via `ShuffleThreadPool` ## Configuration | Config | Description | | ----------------------------------------------- | ----------------------------------- | | `spark.comet.columnar.shuffle.async.enabled` | Enable async spill writes | | `spark.comet.columnar.shuffle.async.thread.num` | Threads per writer for async | | `spark.comet.columnar.shuffle.batch.size` | Rows per Arrow batch | | `spark.comet.columnar.shuffle.spill.threshold` | Row count threshold for spill | | `spark.comet.exec.shuffle.compression.codec` | Compression codec (zstd, lz4, etc.) | ================================================ FILE: docs/source/contributor-guide/native_shuffle.md ================================================ # Native Shuffle This document describes Comet's native shuffle implementation (`CometNativeShuffle`), which performs shuffle operations entirely in Rust code for maximum performance. For the JVM-based alternative, see [JVM Shuffle](jvm_shuffle.md). ## Overview Native shuffle takes columnar input directly from Comet native operators and performs partitioning, encoding, and writing in native Rust code. This avoids the columnar-to-row-to-columnar conversion overhead that JVM shuffle incurs. ``` Comet Native (columnar) → Native Shuffle → Arrow IPC → columnar ``` Compare this to JVM shuffle's data path: ``` Comet Native (columnar) → ColumnarToRowExec → rows → JVM Shuffle → Arrow IPC → columnar ``` ## When Native Shuffle is Used Native shuffle (`CometExchange`) is selected when all of the following conditions are met: 1. **Shuffle mode allows native**: `spark.comet.exec.shuffle.mode` is `native` or `auto`. 2. **Child plan is a Comet native operator**: The child must be a `CometPlan` that produces columnar output. Row-based Spark operators require JVM shuffle. 3. **Supported partitioning type**: Native shuffle supports: - `HashPartitioning` - `RangePartitioning` - `SinglePartition` - `RoundRobinPartitioning` 4. **Supported partition key types**: For `HashPartitioning` and `RangePartitioning`, partition keys must be primitive types. Complex types (struct, array, map) as partition keys require JVM shuffle. Note that complex types are fully supported as data columns in native shuffle. ## Architecture ``` ┌─────────────────────────────────────────────────────────────────────────────┐ │ CometShuffleManager │ │ - Routes to CometNativeShuffleWriter for CometNativeShuffleHandle │ └─────────────────────────────────────────────────────────────────────────────┘ │ ▼ ┌─────────────────────────────────────────────────────────────────────────────┐ │ CometNativeShuffleWriter │ │ - Constructs protobuf operator plan │ │ - Invokes native execution via CometExec.getCometIterator() │ └─────────────────────────────────────────────────────────────────────────────┘ │ ▼ (JNI) ┌─────────────────────────────────────────────────────────────────────────────┐ │ ShuffleWriterExec (Rust) │ │ - DataFusion ExecutionPlan │ │ - Orchestrates partitioning and writing │ └─────────────────────────────────────────────────────────────────────────────┘ │ │ ▼ ▼ ┌───────────────────────────────────┐ ┌───────────────────────────────────┐ │ MultiPartitionShuffleRepartitioner │ │ SinglePartitionShufflePartitioner │ │ (hash/range partitioning) │ │ (single partition case) │ └───────────────────────────────────┘ └───────────────────────────────────┘ │ ▼ ┌───────────────────────────────────┐ │ ShuffleBlockWriter │ │ (Arrow IPC + compression) │ └───────────────────────────────────┘ │ ▼ ┌─────────────────┐ │ Data + Index │ │ Files │ └─────────────────┘ ``` ## Key Classes ### Scala Side | Class | Location | Description | | ------------------------------ | ------------------------------------------------ | --------------------------------------------------------------------------------------------- | | `CometShuffleExchangeExec` | `.../shuffle/CometShuffleExchangeExec.scala` | Physical plan node. Validates types and partitioning, creates `CometShuffleDependency`. | | `CometNativeShuffleWriter` | `.../shuffle/CometNativeShuffleWriter.scala` | Implements `ShuffleWriter`. Builds protobuf plan and invokes native execution. | | `CometShuffleDependency` | `.../shuffle/CometShuffleDependency.scala` | Extends `ShuffleDependency`. Holds shuffle type, schema, and range partition bounds. | | `CometBlockStoreShuffleReader` | `.../shuffle/CometBlockStoreShuffleReader.scala` | Reads shuffle blocks via `ShuffleBlockFetcherIterator`. Decodes Arrow IPC to `ColumnarBatch`. | | `NativeBatchDecoderIterator` | `.../shuffle/NativeBatchDecoderIterator.scala` | Reads compressed Arrow IPC from input stream. Calls native decode via JNI. | ### Rust Side | File | Location | Description | | ----------------------- | ------------------------------------ | ------------------------------------------------------------------------------------ | | `shuffle_writer.rs` | `native/core/src/execution/shuffle/` | `ShuffleWriterExec` plan and partitioners. Main shuffle logic. | | `codec.rs` | `native/core/src/execution/shuffle/` | `ShuffleBlockWriter` for Arrow IPC encoding with compression. Also handles decoding. | | `comet_partitioning.rs` | `native/core/src/execution/shuffle/` | `CometPartitioning` enum defining partition schemes (Hash, Range, Single). | ## Data Flow ### Write Path 1. **Plan construction**: `CometNativeShuffleWriter` builds a protobuf operator plan containing: - A scan operator reading from the input iterator - A `ShuffleWriter` operator with partitioning config and compression codec 2. **Native execution**: `CometExec.getCometIterator()` executes the plan in Rust. 3. **Partitioning**: `ShuffleWriterExec` receives batches and routes to the appropriate partitioner: - `MultiPartitionShuffleRepartitioner`: For hash/range/round-robin partitioning - `SinglePartitionShufflePartitioner`: For single partition (simpler path) 4. **Buffering and spilling**: The partitioner buffers rows per partition. When memory pressure exceeds the threshold, partitions spill to temporary files. 5. **Encoding**: `ShuffleBlockWriter` encodes each partition's data as compressed Arrow IPC: - Writes compression type header - Writes field count header - Writes compressed IPC stream 6. **Output files**: Two files are produced: - **Data file**: Concatenated partition data - **Index file**: Array of 8-byte little-endian offsets marking partition boundaries 7. **Commit**: Back in JVM, `CometNativeShuffleWriter` reads the index file to get partition lengths and commits via Spark's `IndexShuffleBlockResolver`. ### Read Path 1. `CometBlockStoreShuffleReader` fetches shuffle blocks via `ShuffleBlockFetcherIterator`. 2. For each block, `NativeBatchDecoderIterator`: - Reads the 8-byte compressed length header - Reads the 8-byte field count header - Reads the compressed IPC data - Calls `Native.decodeShuffleBlock()` via JNI 3. Native code decompresses and deserializes the Arrow IPC stream. 4. Arrow FFI transfers the `RecordBatch` to JVM as a `ColumnarBatch`. ## Partitioning ### Hash Partitioning Native shuffle implements Spark-compatible hash partitioning: - Uses Murmur3 hash function with seed 42 (matching Spark) - Computes hash of partition key columns - Applies modulo by partition count: `partition_id = hash % num_partitions` ### Range Partitioning For range partitioning: 1. Spark's `RangePartitioner` samples data and computes partition boundaries on the driver. 2. Boundaries are serialized to the native plan. 3. Native code converts sort key columns to comparable row format. 4. Binary search (`partition_point`) determines which partition each row belongs to. ### Single Partition The simplest case: all rows go to partition 0. Uses `SinglePartitionShufflePartitioner` which simply concatenates batches to reach the configured batch size. ### Round Robin Partitioning Comet implements round robin partitioning using hash-based assignment for determinism: 1. Computes a Murmur3 hash of columns (using seed 42) 2. Assigns partitions directly using the hash: `partition_id = hash % num_partitions` This approach guarantees determinism across retries, which is critical for fault tolerance. However, unlike true round robin which cycles through partitions row-by-row, hash-based assignment only provides even distribution when the data has sufficient variation in the hashed columns. Data with low cardinality or identical values may result in skewed partition sizes. ## Memory Management Native shuffle uses DataFusion's memory management with spilling support: - **Memory pool**: Tracks memory usage across the shuffle operation. - **Spill threshold**: When buffered data exceeds the threshold, partitions spill to disk. - **Per-partition spilling**: Each partition has its own spill file. Multiple spills for a partition are concatenated when writing the final output. - **Scratch space**: Reusable buffers for partition ID computation to reduce allocations. The `MultiPartitionShuffleRepartitioner` manages: - `PartitionBuffer`: In-memory buffer for each partition - `SpillFile`: Temporary file for spilled data - Memory tracking via `MemoryConsumer` trait ## Compression Native shuffle supports multiple compression codecs configured via `spark.comet.exec.shuffle.compression.codec`: | Codec | Description | | -------- | ------------------------------------------------------ | | `zstd` | Zstandard compression. Best ratio, configurable level. | | `lz4` | LZ4 compression. Fast with good ratio. | | `snappy` | Snappy compression. Fastest, lower ratio. | | `none` | No compression. | The compression codec is applied uniformly to all partitions. Each partition's data is independently compressed, allowing parallel decompression during reads. ## Configuration | Config | Default | Description | | ------------------------------------------------- | ------- | ---------------------------------------- | | `spark.comet.exec.shuffle.enabled` | `true` | Enable Comet shuffle | | `spark.comet.exec.shuffle.mode` | `auto` | Shuffle mode: `native`, `jvm`, or `auto` | | `spark.comet.exec.shuffle.compression.codec` | `zstd` | Compression codec | | `spark.comet.exec.shuffle.compression.zstd.level` | `1` | Zstd compression level | | `spark.comet.shuffle.write.buffer.size` | `1MB` | Write buffer size | | `spark.comet.columnar.shuffle.batch.size` | `8192` | Target rows per batch | ## Comparison with JVM Shuffle | Aspect | Native Shuffle | JVM Shuffle | | ------------------- | -------------------------------------- | --------------------------------- | | Input format | Columnar (direct from Comet operators) | Row-based (via ColumnarToRowExec) | | Partitioning logic | Rust implementation | Spark's partitioner | | Supported schemes | Hash, Range, Single, RoundRobin | Hash, Range, Single, RoundRobin | | Partition key types | Primitives only (Hash, Range) | Any type | | Performance | Higher (no format conversion) | Lower (columnar→row→columnar) | | Writer variants | Single path | Bypass (hash) and sort-based | See [JVM Shuffle](jvm_shuffle.md) for details on the JVM-based implementation. ================================================ FILE: docs/source/contributor-guide/parquet_scans.md ================================================ # Comet Parquet Scan Implementations Comet currently has two distinct implementations of the Parquet scan operator. | Scan Implementation | Notes | | ----------------------- | ---------------------- | | `native_datafusion` | Fully native scan | | `native_iceberg_compat` | Hybrid JVM/native scan | The configuration property `spark.comet.scan.impl` is used to select an implementation. The default setting is `spark.comet.scan.impl=auto`, which attempts to use `native_datafusion` first, and falls back to Spark if the scan cannot be converted (e.g., due to unsupported features). Most users should not need to change this setting. However, it is possible to force Comet to use a particular implementation for all scan operations by setting this configuration property to one of the following implementations. For example: `--conf spark.comet.scan.impl=native_datafusion`. The following features are not supported by either scan implementation, and Comet will fall back to Spark in these scenarios: - `ShortType` columns, by default. When reading Parquet files written by systems other than Spark that contain columns with the logical type `UINT_8` (unsigned 8-bit integers), Comet may produce different results than Spark. Spark maps `UINT_8` to `ShortType`, but Comet's Arrow-based readers respect the unsigned type and read the data as unsigned rather than signed. Since Comet cannot distinguish `ShortType` columns that came from `UINT_8` versus signed `INT16`, by default Comet falls back to Spark when scanning Parquet files containing `ShortType` columns. This behavior can be disabled by setting `spark.comet.scan.unsignedSmallIntSafetyCheck=false`. Note that `ByteType` columns are always safe because they can only come from signed `INT8`, where truncation preserves the signed value. - Default values that are nested types (e.g., maps, arrays, structs). Literal default values are supported. - Spark's Datasource V2 API. When `spark.sql.sources.useV1SourceList` does not include `parquet`, Spark uses the V2 API for Parquet scans. The DataFusion-based implementations only support the V1 API. - Spark metadata columns (e.g., `_metadata.file_path`) - No support for Dynamic Partition Pruning (DPP) The following shared limitation may produce incorrect results without falling back to Spark: - No support for datetime rebasing. When reading Parquet files containing dates or timestamps written before Spark 3.0 (which used a hybrid Julian/Gregorian calendar), dates/timestamps will be read as if they were written using the Proleptic Gregorian calendar. This may produce incorrect results for dates before October 15, 1582. The `native_datafusion` scan has some additional limitations, mostly related to Parquet metadata. All of these cause Comet to fall back to Spark (including when using `auto` mode). Note that the `native_datafusion` scan requires `spark.comet.exec.enabled=true` because the scan node must be wrapped by `CometExecRule`. - No support for row indexes - No support for reading Parquet field IDs - No support for `input_file_name()`, `input_file_block_start()`, or `input_file_block_length()` SQL functions. The `native_datafusion` scan does not use Spark's `FileScanRDD`, so these functions cannot populate their values. - No support for `ignoreMissingFiles` or `ignoreCorruptFiles` being set to `true` - Duplicate field names in case-insensitive mode (e.g., a Parquet file with both `B` and `b` columns) are detected at read time and raise a `SparkRuntimeException` with error class `_LEGACY_ERROR_TEMP_2093`, matching Spark's behavior. The `native_iceberg_compat` scan has the following additional limitation that may produce incorrect results without falling back to Spark: - Some Spark configuration values are hard-coded to their defaults rather than respecting user-specified values. This may produce incorrect results when non-default values are set. The affected configurations are `spark.sql.parquet.binaryAsString`, `spark.sql.parquet.int96AsTimestamp`, `spark.sql.caseSensitive`, `spark.sql.parquet.inferTimestampNTZ.enabled`, and `spark.sql.legacy.parquet.nanosAsLong`. See [issue #1816](https://github.com/apache/datafusion-comet/issues/1816) for more details. ## S3 Support The `native_datafusion` and `native_iceberg_compat` Parquet scan implementations completely offload data loading to native code. They use the [`object_store` crate](https://crates.io/crates/object_store) to read data from S3 and support configuring S3 access using standard [Hadoop S3A configurations](https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/index.html#General_S3A_Client_configuration) by translating them to the `object_store` crate's format. This implementation maintains compatibility with existing Hadoop S3A configurations, so existing code will continue to work as long as the configurations are supported and can be translated without loss of functionality. #### Additional S3 Configuration Options Beyond credential providers, the `native_datafusion` and `native_iceberg_compat` implementations support additional S3 configuration options: | Option | Description | | ------------------------------- | -------------------------------------------------------------------------------------------------- | | `fs.s3a.endpoint` | The endpoint of the S3 service | | `fs.s3a.endpoint.region` | The AWS region for the S3 service. If not specified, the region will be auto-detected. | | `fs.s3a.path.style.access` | Whether to use path style access for the S3 service (true/false, defaults to virtual hosted style) | | `fs.s3a.requester.pays.enabled` | Whether to enable requester pays for S3 requests (true/false) | All configuration options support bucket-specific overrides using the pattern `fs.s3a.bucket.{bucket-name}.{option}`. #### Examples The following examples demonstrate how to configure S3 access with the `native_datafusion` and `native_iceberg_compat` Parquet scan implementations using different authentication methods. **Example 1: Simple Credentials** This example shows how to access a private S3 bucket using an access key and secret key. The `fs.s3a.aws.credentials.provider` configuration can be omitted since `org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider` is included in Hadoop S3A's default credential provider chain. ```shell $SPARK_HOME/bin/spark-shell \ ... --conf spark.comet.scan.impl=native_datafusion \ --conf spark.hadoop.fs.s3a.access.key=my-access-key \ --conf spark.hadoop.fs.s3a.secret.key=my-secret-key ... ``` **Example 2: Assume Role with Web Identity Token** This example demonstrates using an assumed role credential to access a private S3 bucket, where the base credential for assuming the role is provided by a web identity token credentials provider. ```shell $SPARK_HOME/bin/spark-shell \ ... --conf spark.comet.scan.impl=native_datafusion \ --conf spark.hadoop.fs.s3a.aws.credentials.provider=org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider \ --conf spark.hadoop.fs.s3a.assumed.role.arn=arn:aws:iam::123456789012:role/my-role \ --conf spark.hadoop.fs.s3a.assumed.role.session.name=my-session \ --conf spark.hadoop.fs.s3a.assumed.role.credentials.provider=com.amazonaws.auth.WebIdentityTokenCredentialsProvider ... ``` #### Limitations The S3 support of `native_datafusion` and `native_iceberg_compat` has the following limitations: 1. **Partial Hadoop S3A configuration support**: Not all Hadoop S3A configurations are currently supported. Only the configurations listed in the tables above are translated and applied to the underlying `object_store` crate. 2. **Custom credential providers**: Custom implementations of AWS credential providers are not supported. The implementation only supports the standard credential providers listed in the table above. We are planning to add support for custom credential providers through a JNI-based adapter that will allow calling Java credential providers from native code. See [issue #1829](https://github.com/apache/datafusion-comet/issues/1829) for more details. ================================================ FILE: docs/source/contributor-guide/plugin_overview.md ================================================ # Comet Plugin Architecture ## Overview The Comet plugin enhances Spark SQL by introducing optimized query execution and shuffle mechanisms leveraging native code. It integrates with Spark's plugin framework and extension API to replace or extend Spark's default behavior. --- # Plugin Components ## Comet SQL Plugin The entry point to Comet is the org.apache.spark.CometPlugin class, which is registered in Spark using the following configuration: ``` --conf spark.plugins=org.apache.spark.CometPlugin ``` The plugin is loaded on the Spark driver and does not provide executor-side plugins. The plugin will update the current `SparkConf` with the extra configuration provided by Comet, such as executor memory configuration. The plugin also registers `CometSparkSessionExtensions` with Spark's extension API. ## CometSparkSessionExtensions On initialization, this class registers two physical plan optimization rules with Spark: `CometScanRule` and `CometExecRule`. These rules run whenever a query stage is being planned during Adaptive Query Execution, and run once for the entire plan when Adaptive Query Execution is disabled. ### CometScanRule `CometScanRule` replaces any Parquet scans with Comet operators. There are different paths for Spark v1 and v2 data sources. When reading from Parquet v1 data sources, Comet replaces `FileSourceScanExec` with a `CometScanExec`, and for v2 data sources, `BatchScanExec` is replaced with `CometBatchScanExec`. In both cases, Comet replaces Spark's Parquet reader with a custom vectorized Parquet reader. This is similar to Spark's vectorized Parquet reader used by the v2 Parquet data source but leverages native code for decoding Parquet row groups directly into Arrow format. Comet only supports a subset of data types and will fall back to Spark's scan if unsupported types exist. Comet can still accelerate the rest of the query execution in this case because `CometSparkToColumnarExec` will convert the output from Spark's scan to Arrow arrays. Note that both `spark.comet.exec.enabled=true` and `spark.comet.convert.parquet.enabled=true` must be set to enable this conversion. Refer to the [Supported Spark Data Types](https://datafusion.apache.org/comet/user-guide/datatypes.html) section in the contributor guide to see a list of currently supported data types. ### CometExecRule This rule traverses bottom-up from the original Spark plan and attempts to replace each operator with a Comet equivalent. For example, a `ProjectExec` will be replaced by `CometProjectExec`. When replacing a node, various checks are performed to determine if Comet can support the operator and its expressions. If an operator, expression, or data type is not supported by Comet then the reason will be stored in a tag on the underlying Spark node and the plan will not be converted. Comet does not support partially replacing subsets of the plan within a query stage because this would involve adding transitions to convert between row-based and columnar data between Spark operators and Comet operators and the overhead of this could outweigh the benefits of running parts of the query stage natively in Comet. ## Query Execution Once the plan has been transformed, any consecutive native Comet operators are combined into a `CometNativeExec` which contains a protocol buffer serialized version of the plan (the serialization code can be found in `QueryPlanSerde`). Spark serializes the physical plan and sends it to the executors when executing tasks. The executors deserialize the plan and invoke it. When `CometNativeExec` is invoked, it will pass the serialized protobuf plan into `Native.createPlan`, which invokes the native code via JNI, where the plan is then deserialized. In the native code there is a `PhysicalPlanner` struct (in `planner.rs`) which converts the deserialized plan into an Apache DataFusion `ExecutionPlan`. In some cases, Comet provides specialized physical operators and expressions to override the DataFusion versions to ensure compatibility with Apache Spark. The leaf nodes in the physical plan are always `ScanExec` and each of these operators will make a JNI call to `CometBatchIterator.next()` to fetch the next input batch. The input could be a Comet native Parquet scan, a Spark exchange, or another native plan. `CometNativeExec` creates a `CometExecIterator` and applies this iterator to the input RDD partitions. Each call to `CometExecIterator.next()` will invoke `Native.executePlan`. Once the plan finishes executing, the resulting Arrow batches are imported into the JVM using Arrow FFI. ## Shuffle Comet integrates with Spark's shuffle mechanism, optimizing both shuffle writes and reads. Comet's shuffle manager must be registered with Spark using the following configuration: ``` --conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager ``` ### Shuffle Writes For shuffle writes, a `ShuffleMapTask` runs in the executors. This task contains a `ShuffleDependency` that is broadcast to all of the executors. It then passes the input RDD to `ShuffleWriteProcessor.write()` which requests a `ShuffleWriter` from the shuffle manager, and this is where it gets a Comet shuffle writer. `ShuffleWriteProcessor` then invokes the dependency RDD and fetches rows/batches and passes them to Comet's shuffle writer, which writes batches to disk in Arrow IPC format. As a result, we cannot avoid having one native plan to produce the shuffle input and another native plan for writing the batches to the shuffle file. ### Shuffle Reads For shuffle reads a `ShuffledRDD` requests a `ShuffleReader` from the shuffle manager. Comet provides a `CometBlockStoreShuffleReader` which is implemented in JVM and fetches blocks from Spark and then creates an `ArrowReaderIterator` to process the blocks using Arrow's `StreamReader` for decoding IPC batches. ================================================ FILE: docs/source/contributor-guide/profiling.md ================================================ # Profiling This guide covers profiling tools and techniques for Comet development. Because Comet spans JVM (Spark) and native (Rust) code, choosing the right profiler depends on what you are investigating. ## Choosing a Profiling Tool | Tool | JVM Frames | Native (Rust) Frames | Install Required | Best For | | ------------------------------------------------------------------------------ | ---------- | -------------------- | ---------------- | ------------------------------------------------------------------------------ | | [async-profiler](https://github.com/async-profiler/async-profiler) | Yes | Yes | Yes | End-to-end Comet profiling with unified JVM + native flame graphs | | [Java Flight Recorder (JFR)](https://docs.oracle.com/en/java/javase/17/jfapi/) | Yes | No | No (JDK 11+) | GC pressure, allocations, thread contention, I/O — any JVM-level investigation | | [cargo-flamegraph](https://github.com/flamegraph-rs/flamegraph) | No | Yes | Yes | Isolated Rust micro-benchmarks without a JVM | **Recommendation:** Use **async-profiler** when profiling Spark queries with Comet enabled — it is the only tool that shows both JVM and native frames in a single flame graph. Use **JFR** when you need rich JVM event data (GC, locks, I/O) without installing anything. Use **cargo-flamegraph** when working on native code in isolation via `cargo bench`. ## Profiling with async-profiler [async-profiler](https://github.com/async-profiler/async-profiler) captures JVM and native code in the same flame graph by using Linux `perf_events` or macOS `dtrace`. This makes it ideal for profiling Comet, where hot paths cross the JNI boundary between Spark and Rust. ### Installation Download a release from the [async-profiler releases page](https://github.com/async-profiler/async-profiler/releases): ```shell # Linux x64 wget https://github.com/async-profiler/async-profiler/releases/download/v3.0/async-profiler-3.0-linux-x64.tar.gz mkdir -p $HOME/opt/async-profiler tar xzf async-profiler-3.0-linux-x64.tar.gz -C $HOME/opt/async-profiler --strip-components=1 export ASYNC_PROFILER_HOME=$HOME/opt/async-profiler ``` On macOS, download the appropriate `macos` archive instead. ### Attaching to a running Spark application Use the `asprof` launcher to attach to a running JVM by PID: ```shell # Start CPU profiling for 30 seconds, output an HTML flame graph $ASYNC_PROFILER_HOME/bin/asprof -d 30 -f flamegraph.html # Wall-clock profiling $ASYNC_PROFILER_HOME/bin/asprof -e wall -d 30 -f flamegraph.html # Start profiling (no duration limit), then stop later $ASYNC_PROFILER_HOME/bin/asprof start -e cpu # ... run your query ... $ASYNC_PROFILER_HOME/bin/asprof stop -f flamegraph.html ``` Find the Spark driver/executor PID with `jps` or `pgrep -f SparkSubmit`. ### Passing profiler flags via spark-submit You can also attach async-profiler as a Java agent at JVM startup: ```shell spark-submit \ --conf "spark.driver.extraJavaOptions=-agentpath:$ASYNC_PROFILER_HOME/lib/libasyncProfiler.so=start,event=cpu,file=driver.html,tree" \ --conf "spark.executor.extraJavaOptions=-agentpath:$ASYNC_PROFILER_HOME/lib/libasyncProfiler.so=start,event=cpu,file=executor.html,tree" \ ... ``` Note: If the executor is distributed then `executor.html` will be written on the remote node. ### Choosing an event type | Event | When to use | | ------- | --------------------------------------------------------------------------------------------------------- | | `cpu` | Default. Shows where CPU cycles are spent. Use for compute-bound queries. | | `wall` | Wall-clock time including blocked/waiting threads. Use to find JNI boundary overhead and I/O stalls. | | `alloc` | Heap allocation profiling. Use to find JVM allocation hotspots around Arrow FFI and columnar conversions. | | `lock` | Lock contention. Use when threads appear to spend time waiting on synchronized blocks or locks. | ### Output formats | Format | Flag | Description | | ---------------- | ------------------ | -------------------------------------------------- | | HTML flame graph | `-f out.html` | Interactive flame graph (default and most useful). | | JFR | `-f out.jfr` | Viewable in JDK Mission Control or IntelliJ. | | Collapsed stacks | `-f out.collapsed` | For use with Brendan Gregg's FlameGraph scripts. | | Text summary | `-o text` | Flat list of hot methods. | ### Platform notes **Linux:** Set `perf_event_paranoid` to allow profiling: ```shell sudo sysctl kernel.perf_event_paranoid=1 # or 0 / -1 for full access sudo sysctl kernel.kptr_restrict=0 # optional: enable kernel symbols ``` **macOS:** async-profiler uses `dtrace` on macOS, which requires running as root or with SIP (System Integrity Protection) adjustments. Native Rust frames may not be fully resolved on macOS; Linux is recommended for the most complete flame graphs. ### Integrated benchmark profiling The TPC benchmark scripts in `benchmarks/tpc/` have built-in async-profiler support via the `--async-profiler` flag. See [benchmarks/tpc/README.md](https://github.com/apache/datafusion-comet/blob/main/benchmarks/tpc/README.md) for details. ## Profiling with Java Flight Recorder [Java Flight Recorder (JFR)](https://docs.oracle.com/en/java/javase/17/jfapi/) is built into JDK 11+ and collects detailed JVM runtime data with very low overhead. It does not see native Rust frames, but is excellent for diagnosing GC pressure, thread contention, I/O latency, and JVM-level allocation patterns. ### Adding JFR flags to spark-submit ```shell spark-submit \ --conf "spark.driver.extraJavaOptions=-XX:StartFlightRecording=duration=120s,filename=driver.jfr" \ --conf "spark.executor.extraJavaOptions=-XX:StartFlightRecording=duration=120s,filename=executor.jfr" \ ... ``` For continuous recording without a fixed duration: ```shell --conf "spark.driver.extraJavaOptions=-XX:StartFlightRecording=disk=true,maxsize=500m,filename=driver.jfr" ``` You can also start and stop recording dynamically using `jcmd`: ```shell jcmd JFR.start name=profile # ... run your query ... jcmd JFR.stop name=profile filename=recording.jfr ``` ### Viewing recordings - **[JDK Mission Control (JMC)](https://jdk.java.net/jmc/)** — the most comprehensive viewer. Shows flame graphs, GC timeline, thread activity, I/O, and allocation hot spots. - **IntelliJ IDEA** — open `.jfr` files directly in the built-in profiler (Run → Open Profiler Snapshot). - **`jfr` CLI** — quick summaries from the command line: `jfr summary driver.jfr` ### Useful JFR events for Comet debugging | Event | What it shows | | ------------------------------------------------------------------- | --------------------------------------------------------------------------- | | `jdk.GCPhasePause` | GC pause durations — helps identify memory pressure from Arrow allocations. | | `jdk.ObjectAllocationInNewTLAB` / `jdk.ObjectAllocationOutsideTLAB` | Allocation hot spots. | | `jdk.JavaMonitorWait` / `jdk.ThreadPark` | Thread contention and lock waits. | | `jdk.FileRead` / `jdk.FileWrite` / `jdk.SocketRead` | I/O latency. | | `jdk.ExecutionSample` | CPU sampling (method profiling, similar to a flame graph). | ### Integrated benchmark profiling The TPC benchmark scripts support `--jfr` for automatic JFR recording during benchmark runs. See [benchmarks/tpc/README.md](https://github.com/apache/datafusion-comet/blob/main/benchmarks/tpc/README.md) for details. ## Profiling Native Code with cargo-flamegraph For profiling Rust code in isolation — without a JVM — use `cargo bench` with [cargo-flamegraph](https://github.com/flamegraph-rs/flamegraph). ### Running micro benchmarks with cargo bench When implementing a new operator or expression, it is good practice to add a new microbenchmark under `core/benches`. It is often easiest to copy an existing benchmark and modify it for the new operator or expression. It is also necessary to add a new section to the `Cargo.toml` file, such as: ```toml [[bench]] name = "shuffle_writer" harness = false ``` These benchmarks are useful for comparing performance between releases or between feature branches and the main branch to help prevent regressions in performance when adding new features or fixing bugs. Individual benchmarks can be run by name with the following command. ```shell cargo bench shuffle_writer ``` Here is some sample output from running this command. ``` Running benches/shuffle_writer.rs (target/release/deps/shuffle_writer-e37b59e37879cce7) Gnuplot not found, using plotters backend shuffle_writer/shuffle_writer time: [2.0880 ms 2.0989 ms 2.1118 ms] Found 9 outliers among 100 measurements (9.00%) 3 (3.00%) high mild 6 (6.00%) high severe ``` ### Profiling with cargo-flamegraph Install cargo-flamegraph: ```shell cargo install flamegraph ``` Follow the instructions in [cargo-flamegraph](https://github.com/flamegraph-rs/flamegraph) for your platform for running flamegraph. Here is a sample command for running `cargo-flamegraph` on MacOS. ```shell cargo flamegraph --root --bench shuffle_writer ``` This will produce output similar to the following. ``` dtrace: system integrity protection is on, some features will not be available dtrace: description 'profile-997 ' matched 1 probe Gnuplot not found, using plotters backend Testing shuffle_writer/shuffle_writer Success dtrace: pid 66402 has exited writing flamegraph to "flamegraph.svg" ``` The generated flamegraph can now be opened in a browser that supports svg format. Here is the flamegraph for this example: ![flamegraph](../_static/images/flamegraph.png) ## Tips for Profiling Comet ### Use wall-clock profiling to spot JNI boundary overhead When profiling Comet with async-profiler, `wall` mode is often more revealing than `cpu` because it captures time spent crossing the JNI boundary, waiting for native results, and blocked on I/O — none of which show up in CPU-only profiles. ```shell $ASYNC_PROFILER_HOME/bin/asprof -e wall -d 60 -f wall-profile.html ``` ### Use alloc profiling around Arrow FFI JVM allocation profiling can identify hotspots in the Arrow FFI path where temporary objects are created during data transfer between JVM and native code: ```shell $ASYNC_PROFILER_HOME/bin/asprof -e alloc -d 60 -f alloc-profile.html ``` Look for allocations in `CometExecIterator`, `CometBatchIterator`, and Arrow vector classes. ### Isolate Rust-only performance issues If a flame graph shows the hot path is entirely within native code, switch to `cargo-flamegraph` to get better symbol resolution and avoid JVM noise: ```shell cd native cargo flamegraph --root --bench ``` ### Correlating JVM and native frames In async-profiler flame graphs, native Rust frames appear below JNI entry points like `Java_org_apache_comet_Native_*`. Look for these transition points to understand how time is split between Spark's JVM code and Comet's native execution. ================================================ FILE: docs/source/contributor-guide/release_process.md ================================================ # Apache DataFusion Comet: Release Process This documentation explains the release process for Apache DataFusion Comet. Some preparation tasks can be performed by any contributor, while certain release tasks can only be performed by a DataFusion Project Management Committee (PMC) member. ## Checklist The following is a quick-reference checklist for the full release process. See the detailed sections below for instructions on each step. - [ ] Release preparation: review expression support status and user guide - [ ] Create release branch - [ ] Generate release documentation - [ ] Update Maven version in release branch - [ ] Update version in main for next development cycle - [ ] Generate the change log and create PR against main - [ ] Cherry-pick the change log commit into the release branch - [ ] Build the jars - [ ] Tag the release candidate - [ ] Update documentation for the new release - [ ] Publish Maven artifacts to staging - [ ] Create the release candidate tarball - [ ] Start the email voting thread - [ ] Once the vote passes: - [ ] Publish source tarball - [ ] Create GitHub release - [ ] Promote Maven artifacts to production - [ ] Push the release tag - [ ] Close the vote and announce the release - [ ] Post release: - [ ] Register the release with Apache Reporter - [ ] Delete old RCs and releases from SVN - [ ] Write a blog post ## Release Preparation Before starting the release process, review the user guide to ensure it accurately reflects the current state of the project: - Review the supported expressions and operators lists in the user guide. Verify that any expressions added since the last release are included and that their support status is accurate. - Spot-check the support status of individual expressions by running tests or queries to confirm they work as documented. - Look for any expressions that may have regressed or changed behavior since the last release and update the documentation accordingly. It is also recommended to run benchmarks (such as TPC-H and TPC-DS) comparing performance against the previous release to check for regressions. See the [Comet Benchmarking Guide](benchmarking.md) for instructions. These are tasks where agentic coding tools can be particularly helpful — for example, scanning the codebase for newly registered expressions and cross-referencing them against the documented list, or generating test queries to verify expression support status. Any issues found should be addressed before creating the release branch. ## Creating the Release Candidate This part of the process can be performed by any committer. Here are the steps, using the 0.13.0 release as an example. ### Create Release Branch This document assumes that GitHub remotes are set up as follows: ```shell $ git remote -v apache git@github.com:apache/datafusion-comet.git (fetch) apache git@github.com:apache/datafusion-comet.git (push) origin git@github.com:yourgithubid/datafusion-comet.git (fetch) origin git@github.com:yourgithubid/datafusion-comet.git (push) ``` Create a release branch from the latest commit in main and push to the `apache` repo: ```shell git fetch apache git checkout main git reset --hard apache/main git checkout -b branch-0.13 git push apache branch-0.13 ``` ### Generate Release Documentation Generate the documentation content for this release. The docs on `main` contain only template markers, so we need to generate the actual content (config tables, compatibility matrices) for the release branch: ```shell ./dev/generate-release-docs.sh git add docs/source/user-guide/latest/ git commit -m "Generate docs for 0.13.0 release" git push apache branch-0.13 ``` This freezes the documentation to reflect the configs and expressions available in this release. ### Update Maven Version Update the `pom.xml` files in the release branch to update the Maven version from `0.13.0-SNAPSHOT` to `0.13.0`. There is no need to update the Rust crate versions because they will already be `0.13.0`. ### Update Version in main Create a PR against the main branch to prepare for developing the next release: - Update the Rust crate version to `0.14.0`. - Update the Maven version to `0.14.0-SNAPSHOT` (both in the `pom.xml` files and also in the diff files under `dev/diffs`). ### Generate the Change Log Generate a change log to cover changes between the previous release and the release branch HEAD by running the provided `dev/release/generate-changelog.py`. It is recommended that you set up a virtual Python environment and then install the dependencies: ```shell cd dev/release python3 -m venv venv source venv/bin/activate pip3 install -r requirements.txt ``` To generate the changelog, set the `GITHUB_TOKEN` environment variable to a valid token and then run the script providing two commit ids or tags followed by the version number of the release being created. The following example generates a change log of all changes between the previous version and the current release branch HEAD revision. ```shell export GITHUB_TOKEN= python3 generate-changelog.py 0.12.0 HEAD 0.13.0 > ../changelog/0.13.0.md ``` Create a PR against the _main_ branch to add this change log and once this is approved and merged, cherry-pick the commit into the release branch. ### Build the jars #### Setup to do the build The build process requires Docker. Download the latest Docker Desktop from https://www.docker.com/products/docker-desktop/. If you have multiple docker contexts running switch to the context of the Docker Desktop. For example - ```shell $ docker context ls NAME DESCRIPTION DOCKER ENDPOINT ERROR default Current DOCKER_HOST based configuration unix:///var/run/docker.sock desktop-linux Docker Desktop unix:///Users/parth/.docker/run/docker.sock my_custom_context * tcp://192.168.64.2:2376 $ docker context use desktop-linux ``` #### Run the build script The `build-release-comet.sh` script will create a docker image for each architecture and use the image to build the platform specific binaries. These builder images are created every time this script is run. The script optionally allows overriding of the repository and branch to build the binaries from (Note that the local git repo is not used in the building of the binaries, but it is used to build the final uber jar). ```shell Usage: build-release-comet.sh [options] This script builds comet native binaries inside a docker image. The image is named "comet-rm" and will be generated by this script Options are: -r [repo] : git repo (default: https://github.com/apache/datafusion-comet.git) -b [branch] : git branch (default: release) -t [tag] : tag for the spark-rm docker image to use for building (default: "latest"). ``` Example: ```shell cd dev/release && ./build-release-comet.sh && cd ../.. ``` #### Build output The build output is installed to a temporary local maven repository. The build script will print the name of the repository location at the end. This location will be required at the time of deploying the artifacts to a staging repository ### Tag the Release Candidate Ensure that the Maven version update and changelog cherry-pick have been pushed to the release branch before tagging. Tag the release branch with `0.13.0-rc1` and push to the `apache` repo ```shell git fetch apache git checkout branch-0.13 git reset --hard apache/branch-0.13 git tag 0.13.0-rc1 git push apache 0.13.0-rc1 ``` Note that pushing a release candidate tag will trigger a GitHub workflow that will build a Docker image and publish it to GitHub Container Registry at https://github.com/apache/datafusion-comet/pkgs/container/datafusion-comet ### Publishing Documentation In `docs` directory: - Update `docs/source/index.rst` and add a new navigation menu link for the new release in the section `_toc.user-guide-links-versioned` - Add a new line to `build.sh` to delete the locally cloned `comet-*` branch for the new release e.g. `comet-0.13` - Update the main method in `generate-versions.py`: ```python latest_released_version = "0.13.0" previous_versions = ["0.11.0", "0.12.0"] ``` Test the documentation build locally, following the instructions in `docs/README.md`. Once verified, create a PR against the main branch with these documentation changes. After merging, the docs will be deployed to https://datafusion.apache.org/comet/ by the documentation publishing workflow. Note that the download links in the installation guide will not work until the release is finalized, but having the documentation available could be useful for anyone testing out the release candidate during the voting period. ## Publishing the Release Candidate This part of the process can mostly only be performed by a PMC member. ### Publish the maven artifacts #### Setup maven ##### One time project setup Setting up your project in the ASF Nexus Repository from here: https://infra.apache.org/publishing-maven-artifacts.html ##### Release Manager Setup Set up your development environment from here: https://infra.apache.org/publishing-maven-artifacts.html ##### Build and publish a release candidate to nexus. The script `publish-to-maven.sh` will publish the artifacts created by the `build-release-comet.sh` script. The artifacts will be signed using the gpg key of the release manager and uploaded to the maven staging repository. Note that installed GPG keys can be listed with `gpg --list-keys`. The gpg key is a 40 character hex string. Note: This script needs `xmllint` to be installed. On macOS xmllint is available by default. On Ubuntu `apt-get install -y libxml2-utils` On RedHat `yum install -y xmlstarlet` ```shell ./dev/release/publish-to-maven.sh -h usage: publish-to-maven.sh options Publish signed artifacts to Maven. Options -u ASF_USERNAME - Username of ASF committer account -r LOCAL_REPO - path to temporary local maven repo (created and written to by 'build-release-comet.sh') The following will be prompted for - ASF_PASSWORD - Password of ASF committer account GPG_KEY - GPG key used to sign release artifacts GPG_PASSPHRASE - Passphrase for GPG key ``` example ```shell ./dev/release/publish-to-maven.sh -u release_manager_asf_id -r /tmp/comet-staging-repo-VsYOX ASF Password : GPG Key (Optional): GPG Passphrase : Creating Nexus staging repository ... ``` In the Nexus repository UI (https://repository.apache.org/) locate and verify the artifacts in staging (https://central.sonatype.org/publish/release/#locate-and-examine-your-staging-repository). The script closes the staging repository but does not release it. Releasing to Maven Central is a manual step performed only after the vote passes (see [Publishing Maven Artifacts](#publishing-maven-artifacts) below). Note that the Maven artifacts are always published under the final release version (e.g. `0.13.0`), not the RC version — the `-rc1` / `-rc2` suffix only appears in the git tag and the source tarball in SVN. Because the script creates a new staging repository on each run, re-staging the same version for a subsequent RC is supported as long as no staging repository for that version has been released to Maven Central. ### Create the Release Candidate Tarball The `create-tarball.sh` script creates a signed source tarball and uploads it to the dev subversion repository. #### Prerequisites Before running this script, ensure you have: 1. A GPG key set up for signing, with your public key uploaded to https://pgp.mit.edu/ 2. Apache SVN credentials (you must be logged into the Apache SVN server) 3. The `requests` Python package installed (`pip3 install requests`) #### Run the script Run the create-tarball script on the release candidate tag (`0.13.0-rc1`): ```shell ./dev/release/create-tarball.sh 0.13.0 1 ``` This will generate an email template for starting the vote. ### Start an Email Voting Thread Send the email that is generated in the previous step to `dev@datafusion.apache.org`. The verification procedure for voters is documented in [Verifying Release Candidates](https://github.com/apache/datafusion-comet/blob/main/dev/release/verifying-release-candidates.md). Voters can also use the `dev/release/verify-release-candidate.sh` script to assist with verification: ```shell ./dev/release/verify-release-candidate.sh 0.13.0 1 ``` ### If the Vote Fails If the vote does not pass, address the issues raised, increment the release candidate number, and repeat from the [Tag the Release Candidate](#tag-the-release-candidate) step. For example, the next attempt would be tagged `0.13.0-rc2`. Before staging the next RC, drop the previous RC's staging repository in the [Nexus UI](https://repository.apache.org/#stagingRepositories) by selecting it and clicking "Drop". This avoids leaving multiple closed staging repositories for the same version and prevents accidentally releasing the wrong one when the vote eventually passes. The Maven version (e.g. `0.13.0`) is shared across all RCs, so each run of `publish-to-maven.sh` creates a new staging repository for the same GAV — only one of them should ever be released to Maven Central. ## Publishing Binary Releases Once the vote passes, we can publish the source and binary releases. ### Publishing Source Tarball Run the release-tarball script to move the tarball to the release subversion repository. ```shell ./dev/release/release-tarball.sh 0.13.0 1 ``` ### Create a release in the GitHub repository Go to https://github.com/apache/datafusion-comet/releases and create a release for the release tag, and paste the changelog in the description. ### Publishing Maven Artifacts Promote the Maven artifacts from staging to production by visiting https://repository.apache.org/#stagingRepositories and selecting the staging repository and then clicking the "release" button. ### Push a release tag to the repo Push a release tag (`0.13.0`) to the `apache` repository. ```shell git fetch apache git checkout 0.13.0-rc1 git tag 0.13.0 git push apache 0.13.0 ``` Note that pushing a release tag will trigger a GitHub workflow that will build a Docker image and publish it to GitHub Container Registry at https://github.com/apache/datafusion-comet/pkgs/container/datafusion-comet Reply to the vote thread to close the vote and announce the release. The announcement email should include: - The release version - A link to the release notes / changelog - A link to the download page or Maven coordinates - Thanks to everyone who contributed and voted ## Post Release ### Register the release Register the release with the [Apache Reporter Service](https://reporter.apache.org/addrelease.html?datafusion) using a version such as `COMET-0.13.0`. ### Delete old RCs and Releases See the ASF documentation on [when to archive](https://www.apache.org/legal/release-policy.html#when-to-archive) for more information. #### Deleting old release candidates from `dev` svn Release candidates should be deleted once the release is published. Get a list of DataFusion Comet release candidates: ```shell svn ls https://dist.apache.org/repos/dist/dev/datafusion | grep comet ``` Delete a release candidate: ```shell svn delete -m "delete old DataFusion Comet RC" https://dist.apache.org/repos/dist/dev/datafusion/apache-datafusion-comet-0.13.0-rc1/ ``` #### Deleting old releases from `release` svn Only the latest release should be available. Delete old releases after publishing the new release. Get a list of DataFusion releases: ```shell svn ls https://dist.apache.org/repos/dist/release/datafusion | grep comet ``` Delete a release: ```shell svn delete -m "delete old DataFusion Comet release" https://dist.apache.org/repos/dist/release/datafusion/datafusion-comet-0.12.0 ``` ### Write a blog post Writing a blog post about the release is a great way to generate more interest in the project. We typically create a Google document where the community can collaborate on a blog post. Once the content is agreed then a PR can be created against the [datafusion-site](https://github.com/apache/datafusion-site) repository to add the blog post. Any contributor can drive this process. ================================================ FILE: docs/source/contributor-guide/roadmap.md ================================================ # Comet Roadmap Comet is an open-source project and contributors are welcome to work on any issues at any time, but we find it helpful to have a roadmap for some of the major items that require coordination between contributors. ## Major Initiatives ### Iceberg Integration Reads of Iceberg tables with Parquet data files are fully native and enabled by default, powered by a scan operator backed by Iceberg-rust ([#2528]). We anticipate major improvements in the next few releases, including bringing Iceberg table format V3 features (_e.g._, encryption) to the reader. [#2528]: https://github.com/apache/datafusion-comet/pull/2528 ### Spark 4.0 Support Comet has experimental support for Spark 4.0, but there is more work to do ([#1637]), such as enabling more Spark SQL tests and fully implementing ANSI support ([#313]) for all supported expressions. [#313]: https://github.com/apache/datafusion-comet/issues/313 [#1637]: https://github.com/apache/datafusion-comet/issues/1637 ### Dynamic Partition Pruning Iceberg table scans support Dynamic Partition Pruning (DPP) filters generated by Spark's `PlanDynamicPruningFilters` optimizer rule ([#3349]). However, we still need to bring this functionality to our Parquet reader. Furthermore, Spark's `PlanAdaptiveDynamicPruningFilters` optimizer rule runs after Comet's rules, so DPP with Adaptive Query Execution requires a redesign of Comet's plan translation. We are focused on implementing DPP to keep Comet competitive with benchmarks that benefit from this feature like TPC-DS. This effort can be tracked at [#3510]. [#3349]: https://github.com/apache/datafusion-comet/pull/3349 [#3510]: https://github.com/apache/datafusion-comet/issues/3510 ## Ongoing Improvements In addition to the major initiatives above, we have the following ongoing areas of work: - Adding support for more Spark expressions - Moving more expressions to the `datafusion-spark` crate in the core DataFusion repository - Performance tuning - Nested type support improvements ================================================ FILE: docs/source/contributor-guide/spark-sql-tests.md ================================================ # Running Spark SQL Tests Running Apache Spark's SQL tests with Comet enabled is a good way to ensure that Comet produces the same results as that version of Spark. To enable this, we apply some changes to the Apache Spark source code so that Comet is enabled when we run the tests. Here is an overview of the changes that we need to make to Spark: - Update the pom.xml to add a dependency on Comet - Modify SparkSession to load the Comet extension - Modify TestHive to load Comet - Modify SQLTestUtilsBase to load Comet when `ENABLE_COMET` environment variable exists Here are the steps involved in running the Spark SQL tests with Comet, using Spark 3.4.3 for this example. ## 1. Install Comet Run `make release` in Comet to install the Comet JAR into the local Maven repository, specifying the Spark version. ```shell PROFILES="-Pspark-3.4" make release ``` ## 2. Clone Spark and Apply Diff Clone Apache Spark locally and apply the diff file from Comet. Note: this is a shallow clone of a tagged Spark commit and is not suitable for general Spark development. ```shell git clone -b 'v3.4.3' --single-branch --depth 1 git@github.com:apache/spark.git apache-spark cd apache-spark git apply ../datafusion-comet/dev/diffs/3.4.3.diff ``` ## 3. Run Spark SQL Tests ### Use the following commands to run the Spark SQL test suite locally. Optionally, enable Comet fallback logging, so that all fallback reasons are logged at `WARN` level. ```shell export ENABLE_COMET_LOG_FALLBACK_REASONS=true ``` ```shell ENABLE_COMET=true ENABLE_COMET_ONHEAP=true build/sbt catalyst/test ENABLE_COMET=true ENABLE_COMET_ONHEAP=true build/sbt "sql/testOnly * -- -l org.apache.spark.tags.ExtendedSQLTest -l org.apache.spark.tags.SlowSQLTest" ENABLE_COMET=true ENABLE_COMET_ONHEAP=true build/sbt "sql/testOnly * -- -n org.apache.spark.tags.ExtendedSQLTest" ENABLE_COMET=true ENABLE_COMET_ONHEAP=true build/sbt "sql/testOnly * -- -n org.apache.spark.tags.SlowSQLTest" ENABLE_COMET=true ENABLE_COMET_ONHEAP=true build/sbt "hive/testOnly * -- -l org.apache.spark.tags.ExtendedHiveTest -l org.apache.spark.tags.SlowHiveTest" ENABLE_COMET=true ENABLE_COMET_ONHEAP=true build/sbt "hive/testOnly * -- -n org.apache.spark.tags.ExtendedHiveTest" ENABLE_COMET=true ENABLE_COMET_ONHEAP=true build/sbt "hive/testOnly * -- -n org.apache.spark.tags.SlowHiveTest" ``` ### Steps to run individual test suites through SBT 1. Open SBT with Comet enabled ```shell ENABLE_COMET=true ENABLE_COMET_ONHEAP=true sbt -J-Xmx4096m -Dspark.test.includeSlowTests=true ``` 2. Run individual tests (Below code runs test named `SPARK-35568` in the `spark-sql` module) ```shell sql/testOnly org.apache.spark.sql.DynamicPartitionPruningV1SuiteAEOn -- -z "SPARK-35568" ``` ### Steps to run individual test suites in IntelliJ IDE 1. Add below configuration in VM Options for your test case (apache-spark repository) ```shell -Dspark.comet.enabled=true -Dspark.comet.debug.enabled=true -Dspark.plugins=org.apache.spark.CometPlugin -DXmx4096m -Dspark.executor.heartbeatInterval=20000 -Dspark.network.timeout=10000 --add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED ``` 2. Set `ENABLE_COMET=true` in environment variables ![img.png](img.png) 3. After the above tests are configured, spark tests can be run with debugging enabled on spark/comet code. Note that Comet is added as a dependency and the classes are readonly while debugging from Spark. Any new changes to Comet are to be built and deployed locally through the command (`PROFILES="-Pspark-3.4" make release`) ## Creating a diff file for a new Spark version Once Comet has support for a new Spark version, we need to create a diff file that can be applied to that version of Apache Spark to enable Comet when running tests. This is a highly manual process and the process can vary depending on the changes in the new version of Spark, but here is a general guide to the process. We typically start by applying a patch from a previous version of Spark. For example, when enabling the tests for Spark version 3.5.6 we may start by applying the existing diff for 3.5.5 first. ```shell cd git/apache/spark git checkout v3.5.6 git apply --reject --whitespace=fix ../datafusion-comet/dev/diffs/3.5.5.diff ``` Any changes that cannot be cleanly applied will instead be written out to reject files. For example, the above command generated the following files. ```shell find . -name "*.rej" ./pom.xml.rej ./sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala.rej ./sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala.rej ./sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala.rej ./sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala.rej ./sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala.rej ./sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala.rej ./sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/binaryfile/BinaryFileFormatSuite.scala.rej ./sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRebaseDatetimeSuite.scala.rej ./sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala.rej ./sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala.rej ./sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala.rej ./sql/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala.rej ./sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala.rej ./sql/core/src/test/scala/org/apache/spark/sql/sources/DisableUnnecessaryBucketedScanSuite.scala.rej ./sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionErrorsSuite.scala.rej ./sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala.rej ./sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala.rej ``` The changes in these reject files need to be applied manually. One method is to use the [wiggle](https://github.com/neilbrown/wiggle) command (`brew install wiggle` on Mac). For example: ```shell wiggle --replace ./sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala ./sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala.rej ``` ## Generating The Diff File The diff file can be generated using the `git diff` command. It may be necessary to set the `core.abbrev` configuration setting to use 11 digits hashes for consistency with existing diff files. Note that there is an `IgnoreComet.scala` that is not part of the Spark codebase, and therefore needs to be added using `git add` before generating the diff. ```shell git config core.abbrev 11; git add sql/core/src/test/scala/org/apache/spark/sql/IgnoreComet.scala git diff v3.5.6 > ../datafusion-comet/dev/diffs/3.5.6.diff ``` ## Running Tests in CI The easiest way to run the tests is to create a PR against Comet and let CI run the tests. When working with a new Spark version, the `spark_sql_test.yaml` and `spark_sql_test_ansi.yaml` files will need updating with the new version. ================================================ FILE: docs/source/contributor-guide/sql-file-tests.md ================================================ # SQL File Tests `CometSqlFileTestSuite` is a test suite that automatically discovers `.sql` test files and runs each query through both Spark and Comet, comparing results. This provides a lightweight way to add expression and operator test coverage without writing Scala test code. ## Running the tests Run all SQL file tests: ```shell ./mvnw test -Dsuites="org.apache.comet.CometSqlFileTestSuite" -Dtest=none ``` Run a single test file by adding the file name (without `.sql` extension) after the suite name: ```shell ./mvnw test -Dsuites="org.apache.comet.CometSqlFileTestSuite create_named_struct" -Dtest=none ``` This uses ScalaTest's substring matching, so the argument must match part of the test name. Test names follow the pattern `sql-file: expressions//.sql []`. ## Test file location SQL test files live under: ``` spark/src/test/resources/sql-tests/expressions/ ``` Files are organized into category subdirectories: ``` expressions/ aggregate/ -- avg, sum, count, min_max, ... array/ -- array_contains, array_append, get_array_item, ... bitwise/ cast/ conditional/ -- case_when, coalesce, if_expr, ... datetime/ -- date_add, date_diff, unix_timestamp, ... decimal/ hash/ map/ -- get_map_value, map_keys, map_values, ... math/ -- abs, ceil, floor, round, sqrt, ... misc/ -- width_bucket, scalar_subquery, ... string/ -- concat, like, substring, lower, upper, ... struct/ -- create_named_struct, get_struct_field, ... ``` The test suite recursively discovers all `.sql` files in these directories. Each file becomes one or more ScalaTest test cases. ## File format A test file consists of SQL comments, directives, statements, and queries separated by blank lines. Here is a minimal example: ```sql statement CREATE TABLE test_abs(v double) USING parquet statement INSERT INTO test_abs VALUES (1.5), (-2.5), (0.0), (NULL) query SELECT abs(v) FROM test_abs ``` ### Directives Directives are SQL comments at the top of the file that configure how the test runs. #### `Config` Sets a Spark SQL config for all queries in the file. ```sql -- Config: spark.sql.ansi.enabled=true ``` #### `ConfigMatrix` Runs the entire file once per combination of values. Multiple `ConfigMatrix` lines produce a cross product of all combinations. ```sql -- ConfigMatrix: spark.sql.optimizer.inSetConversionThreshold=100,0 ``` This generates two test cases: ``` sql-file: expressions/conditional/in_set.sql [spark.sql.optimizer.inSetConversionThreshold=100] sql-file: expressions/conditional/in_set.sql [spark.sql.optimizer.inSetConversionThreshold=0] ``` Only add a `ConfigMatrix` directive when there is a real reason to run the test under multiple configurations. Do not add `ConfigMatrix` directives speculatively. #### `MinSparkVersion` Skips the file when running on a Spark version older than the specified version. ```sql -- MinSparkVersion: 3.5 ``` ### Statements A `statement` block executes DDL or DML and does not check results. Use this for `CREATE TABLE` and `INSERT` commands. Table names are automatically extracted for cleanup after the test. ```sql statement CREATE TABLE my_table(x int, y double) USING parquet statement INSERT INTO my_table VALUES (1, 2.0), (3, 4.0), (NULL, NULL) ``` ### Queries A `query` block executes a SELECT and compares results between Spark and Comet. The query mode controls how results are validated. #### `query` (default mode) Checks that the query runs natively on Comet (not falling back to Spark) and that results match Spark exactly. ```sql query SELECT abs(v) FROM test_abs ``` #### `query spark_answer_only` Only checks that Comet results match Spark. Does not assert that the query runs natively. Use this for expressions that Comet may not fully support yet but should still produce correct results. ```sql query spark_answer_only SELECT some_expression(v) FROM test_table ``` #### `query tolerance=` Checks results with a numeric tolerance. Useful for floating-point functions where small differences are acceptable. ```sql query tolerance=0.0001 SELECT cos(v) FROM test_trig ``` #### `query expect_fallback()` Asserts that the query falls back to Spark and verifies the fallback reason contains the given string. ```sql query expect_fallback(unsupported expression) SELECT unsupported_func(v) FROM test_table ``` #### `query ignore()` Skips the query entirely. Use this for queries that hit known bugs. The reason should be a link to the tracking GitHub issue. ```sql -- Comet bug: space(-1) causes native crash query ignore(https://github.com/apache/datafusion-comet/issues/3326) SELECT space(n) FROM test_space WHERE n < 0 ``` #### `query expect_error()` Asserts that both Spark and Comet throw an exception containing the given pattern. Use this for ANSI mode tests where invalid operations should throw errors. ```sql -- Config: spark.sql.ansi.enabled=true -- integer overflow should throw in ANSI mode query expect_error(ARITHMETIC_OVERFLOW) SELECT 2147483647 + 1 -- division by zero should throw in ANSI mode query expect_error(DIVIDE_BY_ZERO) SELECT 1 / 0 -- array out of bounds should throw in ANSI mode query expect_error(INVALID_ARRAY_INDEX) SELECT array(1, 2, 3)[10] ``` ## Adding a new test 1. Create a `.sql` file under the appropriate subdirectory in `spark/src/test/resources/sql-tests/expressions/`. Create a new subdirectory if no existing category fits. 2. Add the Apache license header as a SQL comment. 3. Add a `ConfigMatrix` directive only if the test needs to run under multiple configurations (e.g., testing behavior that varies with a specific Spark config). Do not add `ConfigMatrix` directives speculatively. 4. Create tables and insert test data using `statement` blocks. Include edge cases such as `NULL`, boundary values, and negative numbers. 5. Add `query` blocks for each expression or behavior to test. Use the default `query` mode when you expect Comet to run the expression natively. Use `query spark_answer_only` when native execution is not yet expected. 6. Run the tests to verify: ```shell ./mvnw test -Dsuites="org.apache.comet.CometSqlFileTestSuite" -Dtest=none ``` ### Tips for writing thorough tests #### Cover all combinations of literal and column arguments Comet often uses different code paths for literal values versus column references. Tests should exercise both. For a function with multiple arguments, test every useful combination. For a single-argument function, test both a column reference and a literal: ```sql -- column argument (reads from Parquet, goes through columnar evaluation) query SELECT ascii(s) FROM test_ascii -- literal arguments query SELECT ascii('A'), ascii(''), ascii(NULL) ``` For a multi-argument function like `concat_ws(sep, str1, str2, ...)`, test with the separator as a column versus a literal, and similarly for the other arguments: ```sql -- all columns query SELECT concat_ws(sep, a, b) FROM test_table -- literal separator, column values query SELECT concat_ws(',', a, b) FROM test_table -- all literals query SELECT concat_ws(',', 'hello', 'world') ``` **Note on constant folding:** Normally Spark constant-folds all-literal expressions during planning, so Comet would never see them. However, `CometSqlFileTestSuite` automatically disables constant folding (by excluding `ConstantFolding` from the optimizer rules), so all-literal queries are evaluated by Comet's native engine. This means you can use the default `query` mode for all-literal cases and they will be tested natively just like column-based queries. #### Cover edge cases Include edge-case values in your test data. The exact cases depend on the function, but common ones include: - **NULL values** -- every test should include NULLs - **Empty strings** -- for string functions - **Zero, negative, and very large numbers** -- for numeric functions - **Boundary values** -- `INT_MIN`, `INT_MAX`, `NaN`, `Infinity`, `-Infinity` for numeric types - **Special characters and multibyte UTF-8** -- for string functions (e.g. `'é'`, `'中文'`, `'\t'`) - **Empty arrays/maps** -- for collection functions - **Single-element and multi-element collections** -- for aggregate and collection functions #### One file per expression Keep each `.sql` file focused on a single expression or a small group of closely related expressions. This makes failures easy to locate and keeps files readable. #### Use comments to label sections Add SQL comments before query blocks to describe what aspect of the expression is being tested. This helps reviewers and future maintainers understand the intent: ```sql -- literal separator with NULL values query SELECT concat_ws(',', NULL, 'b', 'c') -- empty separator query SELECT concat_ws('', a, b, c) FROM test_table ``` ### Using agentic coding tools Writing thorough SQL test files is a task well suited to agentic coding tools such as Claude Code. You can point the tool at an existing test file as an example, describe the expression you want to test, and ask it to generate a complete `.sql` file covering all argument combinations and edge cases. This is significantly faster than writing the combinatorial test cases by hand and helps ensure nothing is missed. For example: ``` Read the test file spark/src/test/resources/sql-tests/expressions/string/ascii.sql and the documentation in docs/source/contributor-guide/sql-file-tests.md. Then write a similar test file for the `reverse` function, covering column arguments, literal arguments, NULLs, empty strings, and multibyte characters. ``` ## Handling test failures When a query fails due to a known Comet bug: 1. File a GitHub issue describing the problem. 2. Change the query mode to `ignore(...)` with a link to the issue. 3. Optionally add a SQL comment above the query explaining the problem. ```sql -- GetArrayItem returns incorrect results with dynamic index query ignore(https://github.com/apache/datafusion-comet/issues/3332) SELECT arr[idx] FROM test_get_array_item ``` When the bug is fixed, remove the `ignore(...)` and restore the original query mode. ## Architecture The test infrastructure consists of two Scala files: - **`SqlFileTestParser`** (`spark/src/test/scala/org/apache/comet/SqlFileTestParser.scala`) -- Parses `.sql` files into a `SqlTestFile` data structure containing directives, statements, and queries. - **`CometSqlFileTestSuite`** (`spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala`) -- Discovers test files at suite initialization time, generates ScalaTest test cases for each file and config combination, and executes them using `CometTestBase` assertion methods. Tables created in test files are automatically cleaned up after each test. ================================================ FILE: docs/source/contributor-guide/sql_error_propagation.md ================================================ # ANSI SQL Error Propagation in Comet ## Overview Apache Comet is a native query accelerator for Apache Spark. It runs SQL expressions in **Rust** (via Apache DataFusion) instead of the JVM, which is much faster. But there's a catch: when something goes wrong — say, a divide-by-zero or a type cast failure — the error needs to travel from Rust code all the way back to the Spark/Scala/Java world as a proper Spark exception with the right type, the right message, and even a pointer to the exact character in the original SQL query where the error happened. This document explains the end-to-end error-propagation pipeline. --- ## The Big Picture ![Error propagation pipeline overview](./error_pipeline_overview.svg) ``` SQL Query (Spark/Scala) │ │ 1. Spark serializes the plan + query context into Protobuf ▼ Protobuf bytes ──────────────────────────────────► JNI boundary │ │ 2. Rust deserializes the plan and registers query contexts ▼ Native execution (DataFusion / Rust) │ │ 3. An error occurs (e.g. divide by zero) ▼ SparkError (Rust enum) │ │ 4. Error is wrapped with SQL location context ▼ SparkErrorWithContext │ │ 5. Serialized to JSON string ▼ JSON string ◄──────────────────────────────────── JNI boundary │ │ 6. Thrown as CometQueryExecutionException ▼ CometExecIterator.scala catches it │ │ 7. JSON is parsed, proper Spark exception is reconstructed ▼ Spark exception (e.g. ArithmeticException with DIVIDE_BY_ZERO errorClass) │ │ 8. Spark displays it to the user with SQL location pointer ▼ User sees: [DIVIDE_BY_ZERO] Division by zero. == SQL (line 1, position 8) == SELECT a/b FROM t ^^^ ``` --- ## Step 1: Spark Serializes Query Context into Protobuf When Spark compiles a SQL query, it parses it and attaches _origin_ information to every expression — the line number, column offset, and the full SQL text. `QueryPlanSerde.scala` is the Scala code that converts Spark's physical execution plan into a Protobuf binary that gets sent to the Rust side. It extracts origin information from each expression and encodes it alongside the expression in the Protobuf payload. ### The `extractQueryContext` function ```scala // spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala private def extractQueryContext(expr: Expression): Option[ExprOuterClass.QueryContext] = { val contexts = expr.origin.getQueryContext // Spark stores context in expr.origin if (contexts != null && contexts.length > 0) { val ctx = contexts(0) ctx match { case sqlCtx: SQLQueryContext => val builder = ExprOuterClass.QueryContext.newBuilder() .setSqlText(sqlCtx.sqlText.getOrElse("")) // full SQL text .setStartIndex(sqlCtx.originStartIndex.getOrElse(...)) // char offset of expression start .setStopIndex(sqlCtx.originStopIndex.getOrElse(...)) // char offset of expression end .setLine(sqlCtx.line.getOrElse(0)) .setStartPosition(sqlCtx.startPosition.getOrElse(0)) // ... Some(builder.build()) } } } ``` Then, for **every single expression** converted to Protobuf, a unique numeric ID and the context are attached: ```scala .map { protoExpr => val builder = protoExpr.toBuilder builder.setExprId(nextExprId()) // unique ID (monotonically increasing counter) extractQueryContext(expr).foreach { ctx => builder.setQueryContext(ctx) // attach the SQL location info } builder.build() } ``` ### The Protobuf schema ```protobuf // native/proto/src/proto/expr.proto message Expr { optional uint64 expr_id = 89; // unique ID for each expression optional QueryContext query_context = 90; // SQL location info // ... actual expression type ... } message QueryContext { string sql_text = 1; // "SELECT a/b FROM t" int32 start_index = 2; // 7 (0-based character index of 'a') int32 stop_index = 3; // 9 (0-based character index of 'b', inclusive) int32 line = 4; // 1 (1-based line number) int32 start_position = 5; // 7 (0-based column position) optional string object_type = 6; // e.g. "VIEW" optional string object_name = 7; // e.g. "v1" } ``` --- ## Step 2: Rust Deserializes the Plan and Registers Query Contexts On the Rust side, `PhysicalPlanner` in `planner.rs` converts the Protobuf into DataFusion's physical plan. A `QueryContextMap` — a global registry — maps expression IDs to their SQL context. ### `QueryContextMap` (`native/spark-expr/src/query_context.rs`) ```rust pub struct QueryContextMap { contexts: RwLock>>, } impl QueryContextMap { pub fn register(&self, expr_id: u64, context: QueryContext) { ... } pub fn get(&self, expr_id: u64) -> Option> { ... } } ``` This is basically a lookup table: "for expression #42, the SQL context is: text=`SELECT a/b FROM t`, characters 7–9, line 1, column 7". ### The `PhysicalPlanner` registers contexts during plan creation ```rust // native/core/src/execution/planner.rs pub struct PhysicalPlanner { query_context_registry: Arc, // ... } pub(crate) fn create_expr(&self, spark_expr: &Expr, ...) { // 1. If this expression has a query context, register it if let (Some(expr_id), Some(ctx_proto)) = (spark_expr.expr_id, spark_expr.query_context.as_ref()) { let query_ctx = QueryContext::new( ctx_proto.sql_text.clone(), ctx_proto.start_index, ctx_proto.stop_index, ... ); self.query_context_registry.register(expr_id, query_ctx); } // 2. When building specific expressions (Cast, CheckOverflow, etc.), // look up the context and pass it to the expression ExprStruct::Cast(expr) => { let query_context = spark_expr.expr_id.and_then(|id| { self.query_context_registry.get(id) }); Ok(Arc::new(Cast::new(child, datatype, options, spark_expr.expr_id, query_context))) } } ``` --- ## Step 3: An Error Occurs During Native Execution During query execution, a Rust expression might encounter something like division by zero. ### The `SparkError` enum (`native/spark-expr/src/error.rs`) This enum contains one variant for every kind of error Spark can produce, with exactly the same error message format as Spark: ```rust pub enum SparkError { #[error("[DIVIDE_BY_ZERO] Division by zero. Use `try_divide` to tolerate divisor \ being 0 and return NULL instead. If necessary set \ \"spark.sql.ansi.enabled\" to \"false\" to bypass this error.")] DivideByZero, #[error("[CAST_INVALID_INPUT] The value '{value}' of the type \"{from_type}\" \ cannot be cast to \"{to_type}\" because it is malformed. ...")] CastInvalidValue { value: String, from_type: String, to_type: String }, // ... 30+ more variants matching Spark's error codes ... } ``` When a divide-by-zero happens, the arithmetic expression creates: ```rust return Err(DataFusionError::External(Box::new(SparkError::DivideByZero))); ``` --- ## Step 4: Error Gets Wrapped with SQL Context The expression wrappers (`CheckedBinaryExpr`, `CheckOverflow`, `Cast`) catch the `SparkError` and attach the SQL context using `SparkErrorWithContext`: ```rust // native/core/src/execution/expressions/arithmetic.rs // CheckedBinaryExpr wraps arithmetic operations impl PhysicalExpr for CheckedBinaryExpr { fn evaluate(&self, batch: &RecordBatch) -> Result<...> { match self.child.evaluate(batch) { Err(DataFusionError::External(e)) if self.query_context.is_some() => { if let Some(spark_error) = e.downcast_ref::() { // Wrap the error with SQL location info let wrapped = SparkErrorWithContext::with_context( spark_error.clone(), Arc::clone(self.query_context.as_ref().unwrap()), ); return Err(DataFusionError::External(Box::new(wrapped))); } Err(DataFusionError::External(e)) } other => other, } } } ``` ### `SparkErrorWithContext` (`native/spark-expr/src/error.rs`) ```rust pub struct SparkErrorWithContext { pub error: SparkError, // the actual error pub context: Option>, // optional SQL location } ``` --- ## Step 5: Error Is Serialized to JSON When DataFusion propagates the error all the way up through the execution engine and it reaches the JNI boundary, `throw_exception()` in `errors.rs` is called. It detects the `SparkErrorWithContext` type and calls `.to_json()` on it: ```rust // native/core/src/errors.rs fn throw_exception(env: &mut JNIEnv, error: &CometError, ...) { match error { CometError::DataFusion { source: DataFusionError::External(e), .. } => { if let Some(spark_err_ctx) = e.downcast_ref::() { // Has SQL context → throw with JSON payload let json = spark_err_ctx.to_json(); env.throw_new("org/apache/comet/exceptions/CometQueryExecutionException", json) } else if let Some(spark_err) = e.downcast_ref::() { // No SQL context → throw with JSON payload (no context field) throw_spark_error_as_json(env, spark_err) } } // ... } } ``` The JSON looks like this for a divide-by-zero in `SELECT a/b FROM t`: ```json { "errorType": "DivideByZero", "errorClass": "DIVIDE_BY_ZERO", "params": {}, "context": { "sqlText": "SELECT a/b FROM t", "startIndex": 7, "stopIndex": 9, "line": 1, "startPosition": 7, "objectType": null, "objectName": null }, "summary": "== SQL (line 1, position 8) ==\nSELECT a/b FROM t\n ^^^" } ``` --- ## Step 6: Java Receives `CometQueryExecutionException` On the Java side, a thin exception class carries the JSON string as its message: ```java // common/src/main/java/org/apache/comet/exceptions/CometQueryExecutionException.java public final class CometQueryExecutionException extends CometNativeException { public CometQueryExecutionException(String jsonMessage) { super(jsonMessage); } public boolean isJsonMessage() { String msg = getMessage(); return msg != null && msg.trim().startsWith("{") && msg.trim().endsWith("}"); } } ``` --- ## Step 7: Scala Converts JSON Back to a Real Spark Exception `CometExecIterator.scala` is the Scala code that drives the native execution. Every time it calls into the native engine for the next batch of data, it catches `CometQueryExecutionException` and converts it: ```scala // spark/src/main/scala/org/apache/comet/CometExecIterator.scala try { nativeUtil.getNextBatch(...) } catch { case e: CometQueryExecutionException => logError(s"Native execution for task $taskAttemptId failed", e) throw SparkErrorConverter.convertToSparkException(e) // converts JSON to real exception } ``` ### `SparkErrorConverter.scala` parses the JSON ```scala // spark/src/main/scala/org/apache/comet/SparkErrorConverter.scala def convertToSparkException(e: CometQueryExecutionException): Throwable = { val json = parse(e.getMessage) val errorJson = json.extract[ErrorJson] // Reconstruct Spark's SQLQueryContext from the embedded context val sparkContext: Array[QueryContext] = errorJson.context match { case Some(ctx) => Array(SQLQueryContext( sqlText = Some(ctx.sqlText), line = Some(ctx.line), startPosition = Some(ctx.startPosition), originStartIndex = Some(ctx.startIndex), originStopIndex = Some(ctx.stopIndex), originObjectType = ctx.objectType, originObjectName = ctx.objectName)) case None => Array.empty } // Delegate to version-specific shim convertErrorType(errorJson.errorType, errorClass, params, sparkContext, summary) } ``` ### `ShimSparkErrorConverter` calls the real Spark API Because Spark's `QueryExecutionErrors` API changes between Spark versions (3.4, 3.5, 4.0), there is a separate implementation per version (in `spark-3.4/`, `spark-3.5/`, `spark-4.0/`). ![Shim pattern for per-version Spark API bridging](./shim_pattern.svg) ```scala // spark/src/main/spark-3.5/org/apache/spark/sql/comet/shims/ShimSparkErrorConverter.scala def convertErrorType(errorType: String, errorClass: String, params: Map[String, Any], context: Array[QueryContext], summary: String): Option[Throwable] = { errorType match { case "DivideByZero" => Some(QueryExecutionErrors.divideByZeroError(sqlCtx(context))) // This is the REAL Spark method that creates the ArithmeticException // with the SQL context pointer. The error message will include // "== SQL (line 1, position 8) ==" etc. case "CastInvalidValue" => Some(QueryExecutionErrors.castingCauseOverflowError(...)) // ... all other error types ... case _ => None // fallback to generic SparkException } } ``` The Spark 3.5 shim vs the Spark 4.0 shim differ in subtle API ways: ```scala // 3.5: binaryArithmeticCauseOverflowError does NOT take functionName case "BinaryArithmeticOverflow" => Some(QueryExecutionErrors.binaryArithmeticCauseOverflowError( params("value1").toString.toShort, params("symbol").toString, params("value2").toString.toShort)) // 4.0: overloadedMethod takes functionName parameter case "BinaryArithmeticOverflow" => Some(QueryExecutionErrors.binaryArithmeticCauseOverflowError( params("value1").toString.toShort, params("symbol").toString, params("value2").toString.toShort, params("functionName").toString)) // extra param in 4.0 ``` --- ## Step 8: The User Sees a Proper Spark Error The final exception that propagates out of Spark looks exactly like what native Spark would produce for the same error, including the ANSI error code and the SQL pointer: ``` org.apache.spark.SparkArithmeticException: [DIVIDE_BY_ZERO] Division by zero. Use `try_divide` to tolerate divisor being 0 and return NULL instead. If necessary set "spark.sql.ansi.enabled" to "false" to bypass this error. == SQL (line 1, position 8) == SELECT a/b FROM t ^^^ ``` --- ## Key Data Structures: A Summary | Structure | Language | File | Purpose | | ------------------------------ | -------- | ----------------------------------- | --------------------------------------------------- | | `QueryContext` (proto) | Protobuf | `expr.proto` | Wire format for SQL location info | | `QueryContext` (Rust) | Rust | `query_context.rs` | In-memory SQL location info | | `QueryContextMap` | Rust | `query_context.rs` | Registry: expr_id → QueryContext | | `SparkError` | Rust | `error.rs` | Typed Rust enum matching all Spark error variants | | `SparkErrorWithContext` | Rust | `error.rs` | SparkError + optional QueryContext | | `CometQueryExecutionException` | Java | `CometQueryExecutionException.java` | JNI transport: carries JSON string | | `SparkErrorConverter` | Scala | `SparkErrorConverter.scala` | Parses JSON, creates real Spark exception | | `ShimSparkErrorConverter` | Scala | `ShimSparkErrorConverter.scala` | Per-Spark-version calls to `QueryExecutionErrors.*` | --- ## Why JSON? Why Not Throw the Right Exception Directly? JNI does not support throwing arbitrary Java exception subclasses from Rust directly — you can only provide a class name and a string message. The class name is fixed (always `CometQueryExecutionException`), but the string payload can carry any structured data. JSON was chosen because: 1. It is self-describing — the receiver can parse it without knowing the structure in advance. 2. It is easy to add new fields without breaking old parsers. 3. It maps cleanly to Scala's case class extraction (`json.extract[ErrorJson]`). 4. All the typed information (error class, parameters, SQL context) can be round-tripped perfectly. The alternative — throwing different Java exception classes from Rust — would require a separate JNI throw path for each of the 30+ error types, which would be much harder to maintain. --- ## Why `SparkError` Has Its Own `error_class()` Method Each `SparkError` variant knows its own ANSI error class code (e.g. `"DIVIDE_BY_ZERO"`, `"CAST_INVALID_INPUT"`). This is used both: - In the JSON payload's `"errorClass"` field (so the Java side can pass it to `SparkException(errorClass = ...)` as a fallback) - In the legacy `exception_class()` method that maps to the right Java exception class (e.g. `"java/lang/ArithmeticException"`) --- ## Diagrams ### End-to-End Pipeline ![Error propagation pipeline overview](./error_pipeline_overview.svg) ### QueryContext Journey: From SQL Text to Error Pointer ![QueryContext journey from SQL parser to error message](./query_context_journey.svg) ### Shim Pattern: Per-Version Spark API Bridging ![Shim pattern for per-version Spark API bridging](./shim_pattern.svg) --- _This document and its diagrams were written by [Claude](https://claude.ai) (Anthropic)._ ================================================ FILE: docs/source/contributor-guide/tracing.md ================================================ # Tracing Tracing can be enabled by setting `spark.comet.tracing.enabled=true`. With this feature enabled, each Spark executor will write a JSON event log file in Chrome's [Trace Event Format]. The file will be written to the executor's current working directory with the filename `comet-event-trace.json`. [Trace Event Format]: https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview?tab=t.0#heading=h.yr4qxyxotyw Additionally, enabling the `jemalloc` feature will enable tracing of native memory allocations. ```shell make release COMET_FEATURES="jemalloc" ``` Example output: ```json { "name": "decodeShuffleBlock", "cat": "PERF", "ph": "B", "pid": 1, "tid": 5, "ts": 10109225730 }, { "name": "decodeShuffleBlock", "cat": "PERF", "ph": "E", "pid": 1, "tid": 5, "ts": 10109228835 }, { "name": "decodeShuffleBlock", "cat": "PERF", "ph": "B", "pid": 1, "tid": 5, "ts": 10109245928 }, { "name": "decodeShuffleBlock", "cat": "PERF", "ph": "E", "pid": 1, "tid": 5, "ts": 10109248843 }, { "name": "executePlan", "cat": "PERF", "ph": "E", "pid": 1, "tid": 5, "ts": 10109350935 }, { "name": "getNextBatch[JVM] stage=2", "cat": "PERF", "ph": "E", "pid": 1, "tid": 5, "ts": 10109367116 }, { "name": "getNextBatch[JVM] stage=2", "cat": "PERF", "ph": "B", "pid": 1, "tid": 5, "ts": 10109479156 }, ``` Traces can be viewed with [Perfetto UI]. [Perfetto UI]: https://ui.perfetto.dev Example trace visualization: ![tracing](../_static/images/tracing.png) ## Analyzing Memory Usage The `analyze_trace` tool parses a trace log and compares jemalloc usage against the sum of per-thread Comet memory pool reservations. This is useful for detecting untracked native memory growth where jemalloc allocations exceed what the memory pools account for. Build and run: ```shell cd native cargo run --bin analyze_trace -- /path/to/comet-event-trace.json ``` The tool reads counter events from the trace log. Because tracing logs metrics per thread, `jemalloc_allocated` is a process-wide value (the same global allocation reported from whichever thread logs it), while `thread_NNN_comet_memory_reserved` values are per-thread pool reservations that are summed to get the total tracked memory. Sample output: ``` === Comet Trace Memory Analysis === Counter events parsed: 193104 Threads with memory pools: 8 Peak jemalloc allocated: 3068.2 MB Peak pool total: 2864.6 MB Peak excess (jemalloc - pool): 364.6 MB WARNING: jemalloc exceeded pool reservation at 138 sampled points: Time (us) jemalloc pool_total excess -------------------------------------------------------------- 179578 210.8 MB 0.1 MB 210.7 MB 429663 420.5 MB 145.1 MB 275.5 MB 1304969 2122.5 MB 1797.2 MB 325.2 MB 21974838 407.0 MB 42.3 MB 364.6 MB 33543599 5.5 MB 0.1 MB 5.3 MB --- Final per-thread pool reservations --- thread_60_comet_memory_reserved: 0.0 MB thread_95_comet_memory_reserved: 0.0 MB thread_96_comet_memory_reserved: 0.0 MB ... Total: 0.0 MB ``` Some excess is expected (jemalloc metadata, fragmentation, non-pool allocations like Arrow IPC buffers). Large or growing excess may indicate memory that is not being tracked by the pool. ## Definition of Labels | Label | Meaning | | -------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | | jvm_heap_used | JVM heap memory usage of live objects for the executor process | | jemalloc_allocated | Native memory usage for the executor process (requires `jemalloc` feature) | | thread_NNN_comet_memory_reserved | Memory reserved by Comet's DataFusion memory pool (summed across all contexts on the thread). NNN is the Rust thread ID. | | thread_NNN_comet_jvm_shuffle | Off-heap memory allocated by Comet for columnar shuffle. NNN is the Rust thread ID. | ================================================ FILE: docs/source/index.md ================================================ # Comet Accelerator for Apache Spark and Apache Iceberg

Star Fork

Apache DataFusion Comet is a high-performance accelerator for Apache Spark, built on top of the powerful [Apache DataFusion] query engine. Comet is designed to significantly enhance the performance of Apache Spark workloads while leveraging commodity hardware and seamlessly integrating with the Spark ecosystem without requiring any code changes. Comet also accelerates Apache Iceberg, when performing Parquet scans from Spark. [Apache DataFusion]: https://datafusion.apache.org ## Run Spark Queries at DataFusion Speeds Comet delivers a performance speedup for many queries, enabling faster data processing and shorter time-to-insights. The following chart shows the time it takes to run the 22 TPC-H queries against 100 GB of data in Parquet format using a single executor with 8 cores. See the [Comet Benchmarking Guide](https://datafusion.apache.org/comet/contributor-guide/benchmarking.html) for details of the environment used for these benchmarks. When using Comet, the overall run time is reduced from 687 seconds to 302 seconds, a 2.2x speedup. ![](_static/images/benchmark-results/0.11.0/tpch_allqueries.png) Here is a breakdown showing relative performance of Spark and Comet for each TPC-H query. ![](_static/images/benchmark-results/0.11.0/tpch_queries_compare.png) These benchmarks can be reproduced in any environment using the documentation in the [Comet Benchmarking Guide](/contributor-guide/benchmarking.md). We encourage you to run your own benchmarks. ## Use Commodity Hardware Comet leverages commodity hardware, eliminating the need for costly hardware upgrades or specialized hardware accelerators, such as GPUs or FPGA. By maximizing the utilization of commodity hardware, Comet ensures cost-effectiveness and scalability for your Spark deployments. ## Spark Compatibility Comet aims for 100% compatibility with all supported versions of Apache Spark, allowing you to integrate Comet into your existing Spark deployments and workflows seamlessly. With no code changes required, you can immediately harness the benefits of Comet's acceleration capabilities without disrupting your Spark applications. ## Tight Integration with Apache DataFusion Comet tightly integrates with the core Apache DataFusion project, leveraging its powerful execution engine. With seamless interoperability between Comet and DataFusion, you can achieve optimal performance and efficiency in your Spark workloads. ## Active Community Comet boasts a vibrant and active community of developers, contributors, and users dedicated to advancing the capabilities of Apache DataFusion and accelerating the performance of Apache Spark. ## Getting Started To get started with Apache DataFusion Comet, follow the [installation instructions](https://datafusion.apache.org/comet/user-guide/installation.html). Join the [DataFusion Slack and Discord channels](https://datafusion.apache.org/contributor-guide/communication.html) to connect with other users, ask questions, and share your experiences with Comet. Follow [Apache DataFusion Comet Overview](https://datafusion.apache.org/comet/about/index.html) to get more detailed information ## Contributing We welcome contributions from the community to help improve and enhance Apache DataFusion Comet. Whether it's fixing bugs, adding new features, writing documentation, or optimizing performance, your contributions are invaluable in shaping the future of Comet. Check out our [contributor guide](https://datafusion.apache.org/comet/contributor-guide/contributing.html) to get started. ```{toctree} :maxdepth: 1 :caption: Index :hidden: Comet Overview User Guide Contributor Guide ASF Links ``` ================================================ FILE: docs/source/user-guide/index.md ================================================ # Comet User Guide ```{toctree} :maxdepth: 2 :caption: User Guides 0.15.0-SNAPSHOT 0.14.x <0.14/index> 0.13.x <0.13/index> 0.12.x <0.12/index> 0.11.x <0.11/index> ``` ================================================ FILE: docs/source/user-guide/latest/compatibility.md ================================================ # Compatibility Guide Comet aims to provide consistent results with the version of Apache Spark that is being used. This guide offers information about areas of functionality where there are known differences. ## Parquet Comet has the following limitations when reading Parquet files: - Comet does not support reading decimals encoded in binary format. - No support for default values that are nested types (e.g., maps, arrays, structs). Literal default values are supported. ## ANSI Mode Comet will fall back to Spark for the following expressions when ANSI mode is enabled. These expressions can be enabled by setting `spark.comet.expression.EXPRNAME.allowIncompatible=true`, where `EXPRNAME` is the Spark expression class name. See the [Comet Supported Expressions Guide](expressions.md) for more information on this configuration setting. - Average (supports all numeric inputs except decimal types) - Cast (in some cases) There is an [epic](https://github.com/apache/datafusion-comet/issues/313) where we are tracking the work to fully implement ANSI support. ## Floating-point Number Comparison Spark normalizes NaN and zero for floating point numbers for several cases. See `NormalizeFloatingNumbers` optimization rule in Spark. However, one exception is comparison. Spark does not normalize NaN and zero when comparing values because they are handled well in Spark (e.g., `SQLOrderingUtil.compareFloats`). But the comparison functions of arrow-rs used by DataFusion do not normalize NaN and zero (e.g., [arrow::compute::kernels::cmp::eq](https://docs.rs/arrow/latest/arrow/compute/kernels/cmp/fn.eq.html#)). So Comet adds additional normalization expression of NaN and zero for comparisons, and may still have differences to Spark in some cases, especially when the data contains both positive and negative zero. This is likely an edge case that is not of concern for many users. If it is a concern, setting `spark.comet.exec.strictFloatingPoint=true` will make relevant operations fall back to Spark. ## Incompatible Expressions Expressions that are not 100% Spark-compatible will fall back to Spark by default and can be enabled by setting `spark.comet.expression.EXPRNAME.allowIncompatible=true`, where `EXPRNAME` is the Spark expression class name. See the [Comet Supported Expressions Guide](expressions.md) for more information on this configuration setting. ### Array Expressions - **ArraysOverlap**: Inconsistent behavior when arrays contain NULL values. [#3645](https://github.com/apache/datafusion-comet/issues/3645), [#2036](https://github.com/apache/datafusion-comet/issues/2036) - **ArrayUnion**: Sorts input arrays before performing the union, while Spark preserves the order of the first array and appends unique elements from the second. [#3644](https://github.com/apache/datafusion-comet/issues/3644) - **SortArray**: Nested arrays with `Struct` or `Null` child values are not supported natively and will fall back to Spark. ### Date/Time Expressions - **Hour, Minute, Second**: Incorrectly apply timezone conversion to TimestampNTZ inputs. TimestampNTZ stores local time without timezone, so no conversion should be applied. These expressions work correctly with Timestamp inputs. [#3180](https://github.com/apache/datafusion-comet/issues/3180) - **TruncTimestamp (date_trunc)**: Produces incorrect results when used with non-UTC timezones. Compatible when timezone is UTC. [#2649](https://github.com/apache/datafusion-comet/issues/2649) ### Struct Expressions - **StructsToJson (to_json)**: Does not support `+Infinity` and `-Infinity` for numeric types (float, double). [#3016](https://github.com/apache/datafusion-comet/issues/3016) ## Regular Expressions Comet uses the Rust regexp crate for evaluating regular expressions, and this has different behavior from Java's regular expression engine. Comet will fall back to Spark for patterns that are known to produce different results, but this can be overridden by setting `spark.comet.expression.regexp.allowIncompatible=true`. ## Window Functions Comet's support for window functions is incomplete and known to be incorrect. It is disabled by default and should not be used in production. The feature will be enabled in a future release. Tracking issue: [#2721](https://github.com/apache/datafusion-comet/issues/2721). ## Round-Robin Partitioning Comet's native shuffle implementation of round-robin partitioning (`df.repartition(n)`) is not compatible with Spark's implementation and is disabled by default. It can be enabled by setting `spark.comet.native.shuffle.partitioning.roundrobin.enabled=true`. **Why the incompatibility exists:** Spark's round-robin partitioning sorts rows by their binary `UnsafeRow` representation before assigning them to partitions. This ensures deterministic output for fault tolerance (task retries produce identical results). Comet uses Arrow format internally, which has a completely different binary layout than `UnsafeRow`, making it impossible to match Spark's exact partition assignments. **Comet's approach:** Instead of true round-robin assignment, Comet implements round-robin as hash partitioning on ALL columns. This achieves the same semantic goals: - **Even distribution**: Rows are distributed evenly across partitions (as long as the hash varies sufficiently - in some cases there could be skew) - **Deterministic**: Same input always produces the same partition assignments (important for fault tolerance) - **No semantic grouping**: Unlike hash partitioning on specific columns, this doesn't group related rows together The only difference is that Comet's partition assignments will differ from Spark's. When results are sorted, they will be identical to Spark. Unsorted results may have different row ordering. ## Cast Cast operations in Comet fall into three levels of support: - **C (Compatible)**: The results match Apache Spark - **I (Incompatible)**: The results may match Apache Spark for some inputs, but there are known issues where some inputs will result in incorrect results or exceptions. The query stage will fall back to Spark by default. Setting `spark.comet.expression.Cast.allowIncompatible=true` will allow all incompatible casts to run natively in Comet, but this is not recommended for production use. - **U (Unsupported)**: Comet does not provide a native version of this cast expression and the query stage will fall back to Spark. - **N/A**: Spark does not support this cast. ### String to Decimal Comet's native `CAST(string AS DECIMAL)` implementation matches Apache Spark's behavior, including: - Leading and trailing ASCII whitespace is trimmed before parsing. - Null bytes (`\u0000`) at the start or end of a string are trimmed, matching Spark's `UTF8String` behavior. Null bytes embedded in the middle of a string produce `NULL`. - Fullwidth Unicode digits (U+FF10–U+FF19, e.g. `１２３.４５`) are treated as their ASCII equivalents, so `CAST('１２３.４５' AS DECIMAL(10,2))` returns `123.45`. - Scientific notation (e.g. `1.23E+5`) is supported. - Special values (`inf`, `infinity`, `nan`) produce `NULL`. ### String to Timestamp Comet's native `CAST(string AS TIMESTAMP)` implementation supports all timestamp formats accepted by Apache Spark, including ISO 8601 date-time strings, date-only strings, time-only strings (`HH:MM:SS`), embedded timezone offsets (e.g. `+07:30`, `GMT-01:00`, `UTC`), named timezone suffixes (e.g. `Europe/Moscow`), and the full Spark timestamp year range (-290308 to 294247). Note that `CAST(string AS DATE)` is only compatible for years between 262143 BC and 262142 AD due to an underlying library limitation. ### Legacy Mode ### Try Mode ### ANSI Mode See the [tracking issue](https://github.com/apache/datafusion-comet/issues/286) for more details. ================================================ FILE: docs/source/user-guide/latest/configs.md ================================================ # Comet Configuration Settings Comet provides the following configuration settings. ## Scan Configuration Settings ## Parquet Reader Configuration Settings ## Query Execution Settings ## Viewing Explain Plan & Fallback Reasons These settings can be used to determine which parts of the plan are accelerated by Comet and to see why some parts of the plan could not be supported by Comet. ## Shuffle Configuration Settings ## Memory & Tuning Configuration Settings ## Development & Testing Settings ## Enabling or Disabling Individual Operators ## Enabling or Disabling Individual Scalar Expressions ## Enabling or Disabling Individual Aggregate Expressions ================================================ FILE: docs/source/user-guide/latest/datasources.md ================================================ # Supported Spark Data Sources ## File Formats ### Parquet When `spark.comet.scan.enabled` is enabled, Parquet scans will be performed natively by Comet if all data types in the schema are supported. When this option is not enabled, the scan will fall back to Spark. In this case, enabling `spark.comet.convert.parquet.enabled` will immediately convert the data into Arrow format, allowing native execution to happen after that, but the process may not be efficient. ### Apache Iceberg Comet accelerates Iceberg scans of Parquet files. See the [Iceberg Guide] for more information. [Iceberg Guide]: iceberg.md ### CSV Comet provides experimental native CSV scan support. When `spark.comet.scan.csv.v2.enabled` is enabled, CSV files are read natively for improved performance. This feature is experimental and performance benefits are workload-dependent. Alternatively, when `spark.comet.convert.csv.enabled` is enabled, data from Spark's CSV reader is immediately converted into Arrow format, allowing native execution to happen after that. ### JSON Comet does not provide native JSON scan, but when `spark.comet.convert.json.enabled` is enabled, data is immediately converted into Arrow format, allowing native execution to happen after that. ## Data Catalogs ### Apache Iceberg See the dedicated [Comet and Iceberg Guide](iceberg.md). ## Supported Storages Comet supports most standard storage systems, such as local file system and object storage. ### HDFS Apache DataFusion Comet native reader seamlessly scans files from remote HDFS for [supported formats](#supported-spark-data-sources) ### Using experimental native DataFusion reader Unlike to native Comet reader the Datafusion reader fully supports nested types processing. This reader is currently experimental only To build Comet with native DataFusion reader and remote HDFS support it is required to have a JDK installed Example: Build a Comet for `spark-3.5` provide a JDK path in `JAVA_HOME` Provide the JRE linker path in `RUSTFLAGS`, the path can vary depending on the system. Typically JRE linker is a part of installed JDK ```shell export JAVA_HOME="/opt/homebrew/opt/openjdk@11" make release PROFILES="-Pspark-3.5" COMET_FEATURES=hdfs RUSTFLAGS="-L $JAVA_HOME/libexec/openjdk.jdk/Contents/Home/lib/server" ``` Start Comet with experimental reader and HDFS support as [described](installation.md/#run-spark-shell-with-comet-enabled) and add additional parameters ```shell --conf spark.comet.scan.impl=native_datafusion \ --conf spark.hadoop.fs.defaultFS="hdfs://namenode:9000" \ --conf spark.hadoop.dfs.client.use.datanode.hostname = true \ --conf dfs.client.use.datanode.hostname = true ``` Query a struct type from Remote HDFS ```shell spark.read.parquet("hdfs://namenode:9000/user/data").show(false) root |-- id: integer (nullable = true) |-- first_name: string (nullable = true) |-- personal_info: struct (nullable = true) | |-- firstName: string (nullable = true) | |-- lastName: string (nullable = true) | |-- ageInYears: integer (nullable = true) 25/01/30 16:50:43 INFO core/src/lib.rs: Comet native library version $COMET_VERSION initialized == Physical Plan == * CometColumnarToRow (2) +- CometNativeScan: (1) (1) CometNativeScan: Output [3]: [id#0, first_name#1, personal_info#4] Arguments: [id#0, first_name#1, personal_info#4] (2) CometColumnarToRow [codegen id : 1] Input [3]: [id#0, first_name#1, personal_info#4] 25/01/30 16:50:44 INFO fs-hdfs-0.1.12/src/hdfs.rs: Connecting to Namenode (hdfs://namenode:9000) +---+----------+-----------------+ |id |first_name|personal_info | +---+----------+-----------------+ |2 |Jane |{Jane, Smith, 34}| |1 |John |{John, Doe, 28} | +---+----------+-----------------+ ``` Verify the native scan type should be `CometNativeScan`. More on [HDFS Reader](https://github.com/apache/datafusion-comet/blob/main/native/hdfs/README.md) ### Local HDFS development - Configure local machine network. Add hostname to `/etc/hosts` ```shell 127.0.0.1 localhost namenode datanode1 datanode2 datanode3 ::1 localhost namenode datanode1 datanode2 datanode3 ``` - Start local HDFS cluster, 3 datanodes, namenode url is `namenode:9000` ```shell docker compose -f kube/local/hdfs-docker-compose.yml up ``` - Check the local namenode is up and running on `http://localhost:9870/dfshealth.html#tab-overview` - Build a project with HDFS support ```shell JAVA_HOME="/opt/homebrew/opt/openjdk@11" make release PROFILES="-Pspark-3.5" COMET_FEATURES=hdfs RUSTFLAGS="-L /opt/homebrew/opt/openjdk@11/libexec/openjdk.jdk/Contents/Home/lib/server" ``` - Run local test ```scala withSQLConf( CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", CometConf.COMET_NATIVE_SCAN_IMPL.key -> CometConf.SCAN_NATIVE_DATAFUSION, SQLConf.USE_V1_SOURCE_LIST.key -> "parquet", "fs.defaultFS" -> "hdfs://namenode:9000", "dfs.client.use.datanode.hostname" -> "true") { val df = spark.read.parquet("/tmp/2") df.show(false) df.explain("extended") } } ``` Or use `spark-shell` with HDFS support as described [above](#using-experimental-native-datafusion-reader) ## S3 #### Root CA Certificates One major difference between Spark and Comet is the mechanism for discovering Root CA Certificates. Spark uses the JVM to read CA Certificates from the Java Trust Store, but native Comet scans use system Root CA Certificates (typically stored in `/etc/ssl/certs` on Linux). These scans will not be able to interact with S3 if the Root CA Certificates are not installed. #### Supported Credential Providers AWS credential providers can be configured using the `fs.s3a.aws.credentials.provider` configuration. The following table shows the supported credential providers and their configuration options: | Credential provider | Description | Supported Options | | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | | `org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider` | Access S3 using access key and secret key | `fs.s3a.access.key`, `fs.s3a.secret.key` | | `org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider` | Access S3 using temporary credentials | `fs.s3a.access.key`, `fs.s3a.secret.key`, `fs.s3a.session.token` | | `org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider` | Access S3 using AWS STS assume role | `fs.s3a.assumed.role.arn`, `fs.s3a.assumed.role.session.name` (optional), `fs.s3a.assumed.role.credentials.provider` (optional) | | `org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider` | Access S3 using EC2 instance profile or ECS task credentials (tries ECS first, then IMDS) | None (auto-detected) | | `org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider`
`com.amazonaws.auth.AnonymousAWSCredentials`
`software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider` | Access S3 without authentication (public buckets only) | None | | `com.amazonaws.auth.EnvironmentVariableCredentialsProvider`
`software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider` | Load credentials from environment variables (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN`) | None | | `com.amazonaws.auth.InstanceProfileCredentialsProvider`
`software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider` | Access S3 using EC2 instance metadata service (IMDS) | None | | `com.amazonaws.auth.ContainerCredentialsProvider`
`software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider`
`com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper` | Access S3 using ECS task credentials | None | | `com.amazonaws.auth.WebIdentityTokenCredentialsProvider`
`software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider` | Authenticate using web identity token file | None | Multiple credential providers can be specified in a comma-separated list using the `fs.s3a.aws.credentials.provider` configuration, just as Hadoop AWS supports. If `fs.s3a.aws.credentials.provider` is not configured, Hadoop S3A's default credential provider chain will be used. All configuration options also support bucket-specific overrides using the pattern `fs.s3a.bucket.{bucket-name}.{option}`. ================================================ FILE: docs/source/user-guide/latest/datatypes.md ================================================ # Supported Spark Data Types Comet supports the following Spark data types. Refer to the [Comet Compatibility Guide] for information about data type support in scans and other operators. [Comet Compatibility Guide]: compatibility.md | Data Type | | ------------ | | Null | | Boolean | | Byte | | Short | | Integer | | Long | | Float | | Double | | Decimal | | String | | Binary | | Date | | Timestamp | | TimestampNTZ | | Struct | | Array | | Map | ================================================ FILE: docs/source/user-guide/latest/expressions.md ================================================ # Supported Spark Expressions Comet supports the following Spark expressions. Expressions that are marked as Spark-compatible will either run natively in Comet and provide the same results as Spark, or will fall back to Spark for cases that would not be compatible. All expressions are enabled by default, but most can be disabled by setting `spark.comet.expression.EXPRNAME.enabled=false`, where `EXPRNAME` is the expression name as specified in the following tables, such as `Length`, or `StartsWith`. See the [Comet Configuration Guide] for a full list of expressions that be disabled. Expressions that are not Spark-compatible will fall back to Spark by default and can be enabled by setting `spark.comet.expression.EXPRNAME.allowIncompatible=true`. ## Conditional Expressions | Expression | SQL | Spark-Compatible? | | ---------- | ------------------------------------------- | ----------------- | | CaseWhen | `CASE WHEN expr THEN expr ELSE expr END` | Yes | | If | `IF(predicate_expr, true_expr, false_expr)` | Yes | ## Predicate Expressions | Expression | SQL | Spark-Compatible? | | ------------------ | ------------- | ----------------- | | And | `AND` | Yes | | EqualTo | `=` | Yes | | EqualNullSafe | `<=>` | Yes | | GreaterThan | `>` | Yes | | GreaterThanOrEqual | `>=` | Yes | | LessThan | `<` | Yes | | LessThanOrEqual | `<=` | Yes | | In | `IN` | Yes | | IsNotNull | `IS NOT NULL` | Yes | | IsNull | `IS NULL` | Yes | | InSet | `IN (...)` | Yes | | Not | `NOT` | Yes | | Or | `OR` | Yes | ## String Functions | Expression | Spark-Compatible? | Compatibility Notes | | --------------- | ----------------- | ---------------------------------------------------------------------------------------------------------- | | Ascii | Yes | | | BitLength | Yes | | | Chr | Yes | | | Concat | Yes | Only string inputs are supported | | ConcatWs | Yes | | | Contains | Yes | | | EndsWith | Yes | | | InitCap | No | Behavior is different in some cases, such as hyphenated names. | | Left | Yes | Length argument must be a literal value | | Length | Yes | | | Like | Yes | | | Lower | No | Results can vary depending on locale and character set. Requires `spark.comet.caseConversion.enabled=true` | | OctetLength | Yes | | | Reverse | Yes | | | RLike | No | Uses Rust regexp engine, which has different behavior to Java regexp engine | | StartsWith | Yes | | | StringInstr | Yes | | | StringRepeat | Yes | Negative argument for number of times to repeat causes exception | | StringReplace | Yes | | | StringLPad | Yes | | | StringRPad | Yes | | | StringSpace | Yes | | | StringTranslate | Yes | | | StringTrim | Yes | | | StringTrimBoth | Yes | | | StringTrimLeft | Yes | | | StringTrimRight | Yes | | | Substring | Yes | | | Upper | No | Results can vary depending on locale and character set. Requires `spark.comet.caseConversion.enabled=true` | ## JSON Functions | Expression | Spark-Compatible? | Compatibility Notes | | ------------- | ----------------- | --------------------------------------------------------------------------------------------- | | GetJsonObject | No | Spark allows single-quoted JSON and unescaped control characters which Comet does not support | ## Date/Time Functions | Expression | SQL | Spark-Compatible? | Compatibility Notes | | -------------- | ---------------------------- | ----------------- | -------------------------------------------------------------------------------------------------------------------------------- | | DateAdd | `date_add` | Yes | | | DateDiff | `datediff` | Yes | | | DateFormat | `date_format` | Yes | Partial support. Only specific format patterns are supported. | | DateSub | `date_sub` | Yes | | | DatePart | `date_part(field, source)` | Yes | Supported values of `field`: `year`/`month`/`week`/`day`/`dayofweek`/`dayofweek_iso`/`doy`/`quarter`/`hour`/`minute` | | Days | `days` | Yes | V2 partition transform. Supports DateType and TimestampType inputs. | | Extract | `extract(field FROM source)` | Yes | Supported values of `field`: `year`/`month`/`week`/`day`/`dayofweek`/`dayofweek_iso`/`doy`/`quarter`/`hour`/`minute` | | FromUnixTime | `from_unixtime` | No | Does not support format, supports only -8334601211038 <= sec <= 8210266876799 | | Hour | `hour` | No | Incorrectly applies timezone conversion to TimestampNTZ inputs ([#3180](https://github.com/apache/datafusion-comet/issues/3180)) | | LastDay | `last_day` | Yes | | | Minute | `minute` | No | Incorrectly applies timezone conversion to TimestampNTZ inputs ([#3180](https://github.com/apache/datafusion-comet/issues/3180)) | | Second | `second` | No | Incorrectly applies timezone conversion to TimestampNTZ inputs ([#3180](https://github.com/apache/datafusion-comet/issues/3180)) | | TruncDate | `trunc` | Yes | | | TruncTimestamp | `date_trunc` | No | Incorrect results in non-UTC timezones ([#2649](https://github.com/apache/datafusion-comet/issues/2649)) | | UnixDate | `unix_date` | Yes | | | UnixTimestamp | `unix_timestamp` | Yes | | | Year | `year` | Yes | | | Month | `month` | Yes | | | DayOfMonth | `day`/`dayofmonth` | Yes | | | DayOfWeek | `dayofweek` | Yes | | | WeekDay | `weekday` | Yes | | | DayOfYear | `dayofyear` | Yes | | | WeekOfYear | `weekofyear` | Yes | | | Quarter | `quarter` | Yes | | ## Math Expressions | Expression | SQL | Spark-Compatible? | Compatibility Notes | | -------------- | --------- | ----------------- | --------------------------------- | | Abs | `abs` | Yes | | | Acos | `acos` | Yes | | | Add | `+` | Yes | | | Asin | `asin` | Yes | | | Atan | `atan` | Yes | | | Atan2 | `atan2` | Yes | | | BRound | `bround` | Yes | | | Ceil | `ceil` | Yes | | | Cos | `cos` | Yes | | | Cosh | `cosh` | Yes | | | Cot | `cot` | Yes | | | Divide | `/` | Yes | | | Exp | `exp` | Yes | | | Expm1 | `expm1` | Yes | | | Floor | `floor` | Yes | | | Hex | `hex` | Yes | | | IntegralDivide | `div` | Yes | | | IsNaN | `isnan` | Yes | | | Log | `log` | Yes | | | Log2 | `log2` | Yes | | | Log10 | `log10` | Yes | | | Multiply | `*` | Yes | | | Pow | `power` | Yes | | | Rand | `rand` | Yes | | | Randn | `randn` | Yes | | | Remainder | `%` | Yes | | | Round | `round` | Yes | | | Signum | `signum` | Yes | | | Sin | `sin` | Yes | | | Sinh | `sinh` | Yes | | | Sqrt | `sqrt` | Yes | | | Subtract | `-` | Yes | | | Tan | `tan` | Yes | | | Tanh | `tanh` | Yes | | | TryAdd | `try_add` | Yes | Only integer inputs are supported | | TryDivide | `try_div` | Yes | Only integer inputs are supported | | TryMultiply | `try_mul` | Yes | Only integer inputs are supported | | TrySubtract | `try_sub` | Yes | Only integer inputs are supported | | UnaryMinus | `-` | Yes | | | Unhex | `unhex` | Yes | | ## Hashing Functions | Expression | Spark-Compatible? | | ----------- | ----------------- | | Md5 | Yes | | Murmur3Hash | Yes | | Sha1 | Yes | | Sha2 | Yes | | XxHash64 | Yes | ## Bitwise Expressions | Expression | SQL | Spark-Compatible? | | ------------ | ---- | ----------------- | | BitwiseAnd | `&` | Yes | | BitwiseCount | | Yes | | BitwiseGet | | Yes | | BitwiseOr | `\|` | Yes | | BitwiseNot | `~` | Yes | | BitwiseXor | `^` | Yes | | ShiftLeft | `<<` | Yes | | ShiftRight | `>>` | Yes | ## Aggregate Expressions | Expression | SQL | Spark-Compatible? | Compatibility Notes | | ------------- | ---------- | ------------------------- | ---------------------------------------------------------------- | | Average | | Yes, except for ANSI mode | | | BitAndAgg | | Yes | | | BitOrAgg | | Yes | | | BitXorAgg | | Yes | | | BoolAnd | `bool_and` | Yes | | | BoolOr | `bool_or` | Yes | | | Corr | | Yes | | | Count | | Yes | | | CovPopulation | | Yes | | | CovSample | | Yes | | | First | | No | This function is not deterministic. Results may not match Spark. | | Last | | No | This function is not deterministic. Results may not match Spark. | | Max | | Yes | | | Min | | Yes | | | StddevPop | | Yes | | | StddevSamp | | Yes | | | Sum | | Yes, except for ANSI mode | | | VariancePop | | Yes | | | VarianceSamp | | Yes | | ## Window Functions ```{warning} Window support is disabled by default due to known correctness issues. Tracking issue: [#2721](https://github.com/apache/datafusion-comet/issues/2721). ``` Comet supports using the following aggregate functions within window contexts with PARTITION BY and ORDER BY clauses. | Expression | Spark-Compatible? | Compatibility Notes | | ---------- | ----------------- | ------------------- | | Count | Yes | | | Max | Yes | | | Min | Yes | | | Sum | Yes | | **Note:** Dedicated window functions such as `rank`, `dense_rank`, `row_number`, `lag`, `lead`, `ntile`, `cume_dist`, `percent_rank`, and `nth_value` are not currently supported and will fall back to Spark. ## Array Expressions | Expression | Spark-Compatible? | Compatibility Notes | | -------------- | ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | ArrayAppend | Yes | | | ArrayCompact | No | | | ArrayContains | Yes | | | ArrayDistinct | Yes | | | ArrayExcept | No | | | ArrayFilter | Yes | Only supports case where function is `IsNotNull` | | ArrayInsert | No | | | ArrayIntersect | No | | | ArrayJoin | No | | | ArrayMax | Yes | | | ArrayMin | Yes | | | ArrayRemove | Yes | | | ArrayRepeat | No | | | ArrayUnion | No | Behaves differently than spark. Comet sorts the input arrays before performing the union, while Spark preserves the order of the first array and appends unique elements from the second. | | ArraysOverlap | No | | | CreateArray | Yes | | | ElementAt | Yes | Input must be an array. Map inputs are not supported. | | Flatten | Yes | | | GetArrayItem | Yes | | ## Map Expressions | Expression | Spark-Compatible? | | ------------- | ----------------- | | GetMapValue | Yes | | MapKeys | Yes | | MapEntries | Yes | | MapValues | Yes | | MapFromArrays | Yes | ## Struct Expressions | Expression | Spark-Compatible? | Compatibility Notes | | -------------------- | ----------------- | ----------------------------------------------------------------------------------------------------------------------- | | CreateNamedStruct | Yes | | | GetArrayStructFields | Yes | | | GetStructField | Yes | | | JsonToStructs | No | Partial support. Requires explicit schema. | | StructsToJson | No | Does not support Infinity/-Infinity for numeric types ([#3016](https://github.com/apache/datafusion-comet/issues/3016)) | ## Conversion Expressions | Expression | Spark-Compatible | Compatibility Notes | | ---------- | ------------------------ | ------------------------------------------------------------------------------------------- | | Cast | Depends on specific cast | See the [Comet Compatibility Guide] for list of supported cast expressions and known issues | ## SortOrder | Expression | Spark-Compatible? | Compatibility Notes | | ---------- | ----------------- | ------------------- | | NullsFirst | Yes | | | NullsLast | Yes | | | Ascending | Yes | | | Descending | Yes | | ## Other | Expression | Spark-Compatible? | Compatibility Notes | | ---------------------------- | ----------------- | --------------------------------------------------------------------------- | | Alias | Yes | | | AttributeReference | Yes | | | BloomFilterMightContain | Yes | | | Coalesce | Yes | | | CheckOverflow | Yes | | | KnownFloatingPointNormalized | Yes | | | Literal | Yes | | | MakeDecimal | Yes | | | MonotonicallyIncreasingID | Yes | | | NormalizeNaNAndZero | Yes | | | PromotePrecision | Yes | | | RegExpReplace | No | Uses Rust regexp engine, which has different behavior to Java regexp engine | | ScalarSubquery | Yes | | | SparkPartitionID | Yes | | | ToPrettyString | Yes | | | UnscaledValue | Yes | | [Comet Configuration Guide]: configs.md [Comet Compatibility Guide]: compatibility.md ================================================ FILE: docs/source/user-guide/latest/iceberg.md ================================================ # Accelerating Apache Iceberg Parquet Scans using Comet ## Native Reader Comet's native Iceberg reader relies on reflection to extract `FileScanTask`s from Iceberg, which are then serialized to Comet's native execution engine (see [PR #2528](https://github.com/apache/datafusion-comet/pull/2528)). The example below uses Spark's package downloader to retrieve Comet 0.14.0 and Iceberg 1.8.1, but Comet has been tested with Iceberg 1.5, 1.7, 1.8, 1.9, and 1.10. The native Iceberg reader is enabled by default. To disable it, set `spark.comet.scan.icebergNative.enabled=false`. ```shell $SPARK_HOME/bin/spark-shell \ --packages org.apache.datafusion:comet-spark-spark3.5_2.12:0.14.0,org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1,org.apache.iceberg:iceberg-core:1.8.1 \ --repositories https://repo1.maven.org/maven2/ \ --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \ --conf spark.sql.catalog.spark_catalog=org.apache.iceberg.spark.SparkCatalog \ --conf spark.sql.catalog.spark_catalog.type=hadoop \ --conf spark.sql.catalog.spark_catalog.warehouse=/tmp/warehouse \ --conf spark.plugins=org.apache.spark.CometPlugin \ --conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \ --conf spark.sql.extensions=org.apache.comet.CometSparkSessionExtensions \ --conf spark.comet.explainFallback.enabled=true \ --conf spark.memory.offHeap.enabled=true \ --conf spark.memory.offHeap.size=2g ``` ### Tuning Comet’s native Iceberg reader supports fetching multiple files in parallel to hide I/O latency with the config `spark.comet.scan.icebergNative.dataFileConcurrencyLimit`. This value defaults to 1 to maintain test behavior on Iceberg Java tests without `ORDER BY` clauses, but we suggest increasing it to values between 2 and 8 based on your workload. ### Supported features The native Iceberg reader supports the following features: **Table specifications:** - Iceberg table spec v1 and v2 (v3 will fall back to Spark) **Schema and data types:** - All primitive types including UUID - Complex types: arrays, maps, and structs - Schema evolution (adding and dropping columns) **Time travel and branching:** - `VERSION AS OF` queries to read historical snapshots - Branch reads for accessing named branches **Delete handling (Merge-On-Read tables):** - Positional deletes - Equality deletes - Mixed delete types **Filter pushdown:** - Equality and comparison predicates (`=`, `!=`, `>`, `>=`, `<`, `<=`) - Logical operators (`AND`, `OR`) - NULL checks (`IS NULL`, `IS NOT NULL`) - `IN` and `NOT IN` list operations - `BETWEEN` operations **Partitioning:** - Standard partitioning with partition pruning - Date partitioning with `days()` transform - Bucket partitioning - Truncate transform - Hour transform **Storage:** - Local filesystem - Hadoop Distributed File System (HDFS) - S3-compatible storage (AWS S3, MinIO) ### REST Catalog Comet's native Iceberg reader also supports REST catalogs. The following example shows how to configure Spark to use a REST catalog with Comet's native Iceberg scan: ```shell $SPARK_HOME/bin/spark-shell \ --packages org.apache.datafusion:comet-spark-spark3.5_2.12:0.14.0,org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1,org.apache.iceberg:iceberg-core:1.8.1 \ --repositories https://repo1.maven.org/maven2/ \ --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \ --conf spark.sql.catalog.rest_cat=org.apache.iceberg.spark.SparkCatalog \ --conf spark.sql.catalog.rest_cat.catalog-impl=org.apache.iceberg.rest.RESTCatalog \ --conf spark.sql.catalog.rest_cat.uri=http://localhost:8181 \ --conf spark.sql.catalog.rest_cat.warehouse=/tmp/warehouse \ --conf spark.plugins=org.apache.spark.CometPlugin \ --conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \ --conf spark.sql.extensions=org.apache.comet.CometSparkSessionExtensions \ --conf spark.comet.explainFallback.enabled=true \ --conf spark.memory.offHeap.enabled=true \ --conf spark.memory.offHeap.size=2g ``` Note that REST catalogs require explicit namespace creation before creating tables: ```scala scala> spark.sql("CREATE NAMESPACE rest_cat.db") scala> spark.sql("CREATE TABLE rest_cat.db.test_table (id INT, name STRING) USING iceberg") scala> spark.sql("INSERT INTO rest_cat.db.test_table VALUES (1, 'Alice'), (2, 'Bob')") scala> spark.sql("SELECT * FROM rest_cat.db.test_table").show() ``` ### Current limitations The following scenarios will fall back to Spark's native Iceberg reader: - Iceberg table spec v3 scans - Iceberg writes (reads are accelerated, writes use Spark) - Tables backed by Avro or ORC data files (only Parquet is accelerated) - Tables partitioned on `BINARY` or `DECIMAL` (with precision >28) columns - Scans with residual filters using `truncate`, `bucket`, `year`, `month`, `day`, or `hour` transform functions (partition pruning still works, but row-level filtering of these transforms falls back) ================================================ FILE: docs/source/user-guide/latest/index.rst ================================================ .. Licensed to the Apache Software Foundation (ASF) under one .. or more contributor license agreements. See the NOTICE file .. distributed with this work for additional information .. regarding copyright ownership. The ASF licenses this file .. to you under the Apache License, Version 2.0 (the .. "License"); you may not use this file except in compliance .. with the License. You may obtain a copy of the License at .. http://www.apache.org/licenses/LICENSE-2.0 .. Unless required by applicable law or agreed to in writing, .. software distributed under the License is distributed on an .. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY .. KIND, either express or implied. See the License for the .. specific language governing permissions and limitations .. under the License. .. image:: /_static/images/DataFusionComet-Logo-Light.png :alt: DataFusion Comet Logo ================================ Comet $COMET_VERSION User Guide ================================ .. _toc.user-guide-links-$COMET_VERSION: .. toctree:: :maxdepth: 1 :caption: Comet $COMET_VERSION User Guide Installing Comet Building From Source Supported Data Sources Supported Data Types Supported Operators Supported Expressions Configuration Settings Compatibility Guide Tuning Guide Metrics Guide Iceberg Guide Kubernetes Guide ================================================ FILE: docs/source/user-guide/latest/installation.md ================================================ # Installing DataFusion Comet ## Prerequisites Make sure the following requirements are met and software installed on your machine. ### Supported Operating Systems - Linux - Apple macOS (Intel and Apple Silicon) ### Supported Spark Versions Comet $COMET_VERSION supports the following versions of Apache Spark. We recommend only using Comet with Spark versions where we currently have both Comet and Spark tests enabled in CI. Other versions may work well enough for development and evaluation purposes. | Spark Version | Java Version | Scala Version | Comet Tests in CI | Spark SQL Tests in CI | | ------------- | ------------ | ------------- | ----------------- | --------------------- | | 3.4.3 | 11/17 | 2.12/2.13 | Yes | Yes | | 3.5.5 | 11/17 | 2.12/2.13 | Yes | No | | 3.5.6 | 11/17 | 2.12/2.13 | Yes | No | | 3.5.7 | 11/17 | 2.12/2.13 | Yes | Yes | | 3.5.8 | 11/17 | 2.12/2.13 | Yes | Yes | Note that we do not test the full matrix of supported Java and Scala versions in CI for every Spark version. Experimental support is provided for the following versions of Apache Spark and is intended for development/testing use only and should not be used in production yet. | Spark Version | Java Version | Scala Version | Comet Tests in CI | Spark SQL Tests in CI | | ------------- | ------------ | ------------- | ----------------- | --------------------- | | 4.0.1 | 17 | 2.13 | Yes | Yes | Note that Comet may not fully work with proprietary forks of Apache Spark such as the Spark versions offered by Cloud Service Providers. ## Using a Published JAR File Comet jar files are available in [Maven Central](https://central.sonatype.com/namespace/org.apache.datafusion) for amd64 and arm64 architectures for Linux. For Apple macOS, it is currently necessary to build from source. For performance reasons, published Comet jar files target baseline CPUs available in modern data centers. For example, the amd64 build uses the `x86-64-v3` target that adds CPU instructions (_e.g._, AVX2) common after 2013. Similarly, the arm64 build uses the `neoverse-n1` target, which is a common baseline for ARM cores found in AWS (Graviton2+), GCP, and Azure after 2019. If the Comet library fails for SIGILL (illegal instruction), please open an issue on the GitHub repository describing your environment, and [build from source] for your target architecture. Here are the direct links for downloading the Comet $COMET_VERSION jar file. Note that these links are not valid if you are viewing the documentation for the latest `SNAPSHOT` release, or for the latest release while it is still going through the release vote. Release candidate jars can be found [here](https://repository.apache.org/#nexus-search;quick~org.apache.datafusion). - [Comet plugin for Spark 3.4 / Scala 2.12](https://repo1.maven.org/maven2/org/apache/datafusion/comet-spark-spark3.4_2.12/$COMET_VERSION/comet-spark-spark3.4_2.12-$COMET_VERSION.jar) - [Comet plugin for Spark 3.4 / Scala 2.13](https://repo1.maven.org/maven2/org/apache/datafusion/comet-spark-spark3.4_2.13/$COMET_VERSION/comet-spark-spark3.4_2.13-$COMET_VERSION.jar) - [Comet plugin for Spark 3.5 / Scala 2.12](https://repo1.maven.org/maven2/org/apache/datafusion/comet-spark-spark3.5_2.12/$COMET_VERSION/comet-spark-spark3.5_2.12-$COMET_VERSION.jar) - [Comet plugin for Spark 3.5 / Scala 2.13](https://repo1.maven.org/maven2/org/apache/datafusion/comet-spark-spark3.5_2.13/$COMET_VERSION/comet-spark-spark3.5_2.13-$COMET_VERSION.jar) - [Comet plugin for Spark 4.0 / Scala 2.13 (Experimental)](https://repo1.maven.org/maven2/org/apache/datafusion/comet-spark-spark4.0_2.13/$COMET_VERSION/comet-spark-spark4.0_2.13-$COMET_VERSION.jar) ## Building from source Refer to the [Building from source] guide for instructions from building Comet from source, either from official source releases, or from the latest code in the GitHub repository. [Building from source]: source.md ## Deploying to Kubernetes See the [Comet Kubernetes Guide](kubernetes.md) guide. ## Run Spark Shell with Comet enabled Make sure `SPARK_HOME` points to the same Spark version as Comet was built for. ```shell export COMET_JAR=spark/target/comet-spark-spark3.5_2.12-$COMET_VERSION.jar $SPARK_HOME/bin/spark-shell \ --jars $COMET_JAR \ --conf spark.driver.extraClassPath=$COMET_JAR \ --conf spark.executor.extraClassPath=$COMET_JAR \ --conf spark.plugins=org.apache.spark.CometPlugin \ --conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \ --conf spark.comet.explainFallback.enabled=true \ --conf spark.memory.offHeap.enabled=true \ --conf spark.memory.offHeap.size=16g ``` ### Verify Comet enabled for Spark SQL query Create a test Parquet source ```scala scala> (0 until 10).toDF("a").write.mode("overwrite").parquet("/tmp/test") ``` Query the data from the test source and check: - INFO message shows the native Comet library has been initialized. - The query plan reflects Comet operators being used for this query instead of Spark ones ```scala scala> spark.read.parquet("/tmp/test").createOrReplaceTempView("t1") scala> spark.sql("select * from t1 where a > 5").explain INFO src/lib.rs: Comet native library initialized == Physical Plan == *(1) ColumnarToRow +- CometFilter [a#14], (isnotnull(a#14) AND (a#14 > 5)) +- CometScan parquet [a#14] Batched: true, DataFilters: [isnotnull(a#14), (a#14 > 5)], Format: CometParquet, Location: InMemoryFileIndex(1 paths)[file:/tmp/test], PartitionFilters: [], PushedFilters: [IsNotNull(a), GreaterThan(a,5)], ReadSchema: struct ``` With the configuration `spark.comet.explainFallback.enabled=true`, Comet will log any reasons that prevent a plan from being executed natively. ```scala scala> Seq(1,2,3,4).toDF("a").write.parquet("/tmp/test.parquet") WARN CometSparkSessionExtensions$CometExecRule: Comet cannot execute some parts of this plan natively because: - LocalTableScan is not supported - WriteFiles is not supported - Execute InsertIntoHadoopFsRelationCommand is not supported ``` ## Additional Configuration Depending on your deployment mode you may also need to set the driver & executor class path(s) to explicitly contain Comet otherwise Spark may use a different class-loader for the Comet components than its internal components which will then fail at runtime. For example: ``` --driver-class-path spark/target/comet-spark-spark3.5_2.12-$COMET_VERSION.jar ``` Some cluster managers may require additional configuration, see ### Memory tuning In addition to Apache Spark memory configuration parameters, Comet introduces additional parameters to configure memory allocation for native execution. See [Comet Memory Tuning](./tuning.md) for details. ================================================ FILE: docs/source/user-guide/latest/kubernetes.md ================================================ # Comet Kubernetes Support ## Comet Docker Images Run the following command from the root of this repository to build the Comet Docker image, or use a [published Docker image](https://hub.docker.com/r/apache/datafusion-comet). ```shell docker build -t apache/datafusion-comet -f kube/Dockerfile . ``` ## Example Spark Submit The exact syntax will vary depending on the Kubernetes distribution, but an example `spark-submit` command can be found [here](https://github.com/apache/datafusion-comet/tree/main/benchmarks). ## Helm chart Install helm Spark operator for Kubernetes ```bash # Add the Helm repository helm repo add spark-operator https://kubeflow.github.io/spark-operator helm repo update # Install the operator into the spark-operator namespace and wait for deployments to be ready helm install spark-operator spark-operator/spark-operator --namespace spark-operator --create-namespace --wait ``` Check the operator is deployed ```bash helm status --namespace spark-operator spark-operator NAME: my-release NAMESPACE: spark-operator STATUS: deployed REVISION: 1 TEST SUITE: None ``` Create example Spark application file `spark-pi.yaml` ```bash apiVersion: sparkoperator.k8s.io/v1beta2 kind: SparkApplication metadata: name: spark-pi namespace: default spec: type: Scala mode: cluster image: apache/datafusion-comet:0.7.0-spark3.5.5-scala2.12-java11 imagePullPolicy: IfNotPresent mainClass: org.apache.spark.examples.SparkPi mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.12-3.5.5.jar sparkConf: "spark.executor.extraClassPath": "/opt/spark/jars/comet-spark-spark3.5_2.12-0.7.0.jar" "spark.driver.extraClassPath": "/opt/spark/jars/comet-spark-spark3.5_2.12-0.7.0.jar" "spark.plugins": "org.apache.spark.CometPlugin" "spark.comet.enabled": "true" "spark.comet.exec.enabled": "true" "spark.comet.exec.shuffle.enabled": "true" "spark.comet.exec.shuffle.mode": "auto" "spark.shuffle.manager": "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager" sparkVersion: 3.5.6 driver: labels: version: 3.5.6 cores: 1 coreLimit: 1200m memory: 512m serviceAccount: spark-operator-spark executor: labels: version: 3.5.6 instances: 1 cores: 1 coreLimit: 1200m memory: 512m ``` Refer to [Comet builds](#comet-docker-images) Run Apache Spark application with Comet enabled ```bash kubectl apply -f spark-pi.yaml sparkapplication.sparkoperator.k8s.io/spark-pi created ``` Check application status ```bash kubectl get sparkapp spark-pi NAME STATUS ATTEMPTS START FINISH AGE spark-pi RUNNING 1 2025-03-18T21:19:48Z 65s ``` To check more runtime details ```bash kubectl describe sparkapplication spark-pi .... Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal SparkApplicationSubmitted 8m15s spark-application-controller SparkApplication spark-pi was submitted successfully Normal SparkDriverRunning 7m18s spark-application-controller Driver spark-pi-driver is running Normal SparkExecutorPending 7m11s spark-application-controller Executor [spark-pi-68732195ab217303-exec-1] is pending Normal SparkExecutorRunning 7m10s spark-application-controller Executor [spark-pi-68732195ab217303-exec-1] is running Normal SparkExecutorCompleted 7m5s spark-application-controller Executor [spark-pi-68732195ab217303-exec-1] completed Normal SparkDriverCompleted 7m4s spark-application-controller Driver spark-pi-driver completed ``` Get Driver Logs ```bash kubectl logs spark-pi-driver ``` More info on [Kube Spark operator](https://www.kubeflow.org/docs/components/spark-operator/getting-started/) ================================================ FILE: docs/source/user-guide/latest/metrics.md ================================================ # Comet Metrics ## Spark SQL Metrics Set `spark.comet.metrics.detailed=true` to see all available Comet metrics. ### CometScanExec | Metric | Description | | ----------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `scan time` | Total time to scan a Parquet file. This is not comparable to the same metric in Spark because Comet's scan metric is more accurate. Although both Comet and Spark measure the time in nanoseconds, Spark rounds this time to the nearest millisecond per batch and Comet does not. | ### Exchange Comet adds some additional metrics: | Metric | Description | | ------------------------------- | ------------------------------------------------------------- | | `native shuffle time` | Total time in native code excluding any child operators. | | `repartition time` | Time to repartition batches. | | `memory pool time` | Time interacting with memory pool. | | `encoding and compression time` | Time to encode batches in IPC format and compress using ZSTD. | ## Native Metrics Setting `spark.comet.explain.native.enabled=true` will cause native plans to be logged in each executor. Metrics are logged for each native plan (and there is one plan per task, so this is very verbose). Here is a guide to some of the native metrics. ### ScanExec | Metric | Description | | ----------------- | --------------------------------------------------------------------------------------------------- | | `elapsed_compute` | Total time spent in this operator, fetching batches from a JVM iterator. | | `jvm_fetch_time` | Time spent in the JVM fetching input batches to be read by this `ScanExec` instance. | | `arrow_ffi_time` | Time spent using Arrow FFI to create Arrow batches from the memory addresses returned from the JVM. | ### ShuffleWriterExec | Metric | Description | | ----------------- | ------------------------------------------------------------- | | `elapsed_compute` | Total time excluding any child operators. | | `repart_time` | Time to repartition batches. | | `ipc_time` | Time to encode batches in IPC format and compress using ZSTD. | | `mempool_time` | Time interacting with memory pool. | | `write_time` | Time spent writing bytes to disk. | ================================================ FILE: docs/source/user-guide/latest/operators.md ================================================ # Supported Spark Operators The following Spark operators are currently replaced with native versions. Query stages that contain any operators not supported by Comet will fall back to regular Spark execution. | Operator | Spark-Compatible? | Compatibility Notes | | --------------------------------- | ----------------- | ------------------------------------------------------------------------------------------------------------------ | | BatchScanExec | Yes | Supports Parquet files and Apache Iceberg Parquet scans. See the [Comet Compatibility Guide] for more information. | | BroadcastExchangeExec | Yes | | | BroadcastHashJoinExec | Yes | | | ExpandExec | Yes | | | FileSourceScanExec | Yes | Supports Parquet files. See the [Comet Compatibility Guide] for more information. | | FilterExec | Yes | | | GenerateExec | Yes | Supports `explode` generator only. | | GlobalLimitExec | Yes | | | HashAggregateExec | Yes | | | InsertIntoHadoopFsRelationCommand | No | Experimental support for native Parquet writes. Disabled by default. | | LocalLimitExec | Yes | | | LocalTableScanExec | No | Experimental and disabled by default. | | ObjectHashAggregateExec | Yes | Supports a limited number of aggregates, such as `bloom_filter_agg`. | | ProjectExec | Yes | | | ShuffleExchangeExec | Yes | | | ShuffledHashJoinExec | Yes | | | SortExec | Yes | | | SortMergeJoinExec | Yes | | | UnionExec | Yes | | | WindowExec | No | Disabled by default due to known correctness issues. | [Comet Compatibility Guide]: compatibility.md ================================================ FILE: docs/source/user-guide/latest/source.md ================================================ # Building Comet From Source It is sometimes preferable to build from source for a specific platform. ## Using a Published Source Release Official source releases can be downloaded from https://dist.apache.org/repos/dist/release/datafusion/ ```console # Pick the latest version export COMET_VERSION=$COMET_VERSION # Download the tarball curl -O "https://dist.apache.org/repos/dist/release/datafusion/datafusion-comet-$COMET_VERSION/apache-datafusion-comet-$COMET_VERSION.tar.gz" # Unpack tar -xzf apache-datafusion-comet-$COMET_VERSION.tar.gz cd apache-datafusion-comet-$COMET_VERSION ``` Build ```console make release-nogit PROFILES="-Pspark-3.5" ``` ## Building from the GitHub repository Clone the repository: ```console git clone https://github.com/apache/datafusion-comet.git ``` Build Comet for a specific Spark version: ```console cd datafusion-comet make release PROFILES="-Pspark-3.5" ``` Note that the project builds for Scala 2.12 by default but can be built for Scala 2.13 using an additional profile: ```console make release PROFILES="-Pspark-3.5 -Pscala-2.13" ``` To build Comet from the source distribution on an isolated environment without an access to `github.com` it is necessary to disable `git-commit-id-maven-plugin`, otherwise you will face errors that there is no access to the git during the build process. In that case you may use: ```console make release-nogit PROFILES="-Pspark-3.5" ``` ================================================ FILE: docs/source/user-guide/latest/tuning.md ================================================ # Comet Tuning Guide Comet provides some tuning options to help you get the best performance from your queries. ## Configuring Tokio Runtime Comet uses a global tokio runtime per executor process using tokio's defaults of one worker thread per core and a maximum of 512 blocking threads. These values can be overridden using the environment variables `COMET_WORKER_THREADS` and `COMET_MAX_BLOCKING_THREADS`. It is recommended that `COMET_WORKER_THREADS` be set to the number of executor cores. This may not be necessary in some environments, such as Kubernetes, where the number of cores allocated to a pod will already be equal to the number of executor cores. ## Memory Tuning It is necessary to specify how much memory Comet can use in addition to memory already allocated to Spark. In some cases, it may be possible to reduce the amount of memory allocated to Spark so that overall memory allocation is the same or lower than the original configuration. In other cases, enabling Comet may require allocating more memory than before. See the [Determining How Much Memory to Allocate] section for more details. [Determining How Much Memory to Allocate]: #determining-how-much-memory-to-allocate ### Configuring Comet Memory Comet shares an off-heap memory pool with Spark. The size of the pool is specified by `spark.memory.offHeap.size`. Comet's memory accounting isn't 100% accurate and this can result in Comet using more memory than it reserves, leading to out-of-memory exceptions. To work around this issue, it is possible to set `spark.comet.exec.memoryPool.fraction` to a value less than `1.0` to restrict the amount of memory that can be reserved by Comet. For more details about Spark off-heap memory mode, please refer to [Spark documentation]. [Spark documentation]: https://spark.apache.org/docs/latest/configuration.html Comet implements multiple memory pool implementations. The type of pool can be specified with `spark.comet.exec.memoryPool`. The valid pool types are: - `fair_unified` (default when `spark.memory.offHeap.enabled=true` is set) - `greedy_unified` Both pool types are shared across all native execution contexts within the same Spark task. When Comet executes a shuffle, it runs two native execution contexts concurrently (e.g. one for pre-shuffle operators and one for the shuffle writer). The shared pool ensures that the combined memory usage stays within the per-task limit. The `fair_unified` pool prevents operators from using more than an even fraction of the available memory (i.e. `pool_size / num_reservations`). This pool works best when you know beforehand the query has multiple operators that will likely all need to spill. Sometimes it will cause spills even when there is sufficient memory in order to leave enough memory for other operators. The `greedy_unified` pool type implements a greedy first-come first-serve limit. This pool works well for queries that do not need to spill or have a single spillable operator. [shuffle]: #shuffle [Advanced Memory Tuning]: #advanced-memory-tuning ### Determining How Much Memory to Allocate Generally, increasing the amount of memory allocated to Comet will improve query performance by reducing the amount of time spent spilling to disk, especially for aggregate, join, and shuffle operations. Allocating insufficient memory can result in out-of-memory errors. This is no different from allocating memory in Spark and the amount of memory will vary for different workloads, so some experimentation will be required. Here is a real-world example, based on running benchmarks derived from TPC-H, running on a single executor against local Parquet files using the 100 GB data set. Baseline Spark Performance - Spark completes the benchmark in 632 seconds with 8 cores and 8 GB RAM - With less than 8 GB RAM, performance degrades due to spilling - Spark can complete the benchmark with as little as 3 GB of RAM, but with worse performance (744 seconds) Comet Performance - Comet requires at least 5 GB of RAM, but performance at this level is around 340 seconds, which is significantly faster than Spark with any amount of RAM - Comet running in off-heap with 8 cores completes the benchmark in 295 seconds, more than 2x faster than Spark - It is worth noting that running Comet with only 4 cores and 4 GB RAM completes the benchmark in 520 seconds, providing better performance than Spark for half the resource It may be possible to reduce Comet's memory overhead by reducing batch sizes or increasing number of partitions. ## Optimizing Sorting on Floating-Point Values Sorting on floating-point data types (or complex types containing floating-point values) is not compatible with Spark if the data contains both zero and negative zero. This is likely an edge case that is not of concern for many users and sorting on floating-point data can be enabled by setting `spark.comet.expression.SortOrder.allowIncompatible=true`. ## Optimizing Joins Spark often chooses `SortMergeJoin` over `ShuffledHashJoin` for stability reasons. If the build-side of a `ShuffledHashJoin` is very large then it could lead to OOM in Spark. Vectorized query engines tend to perform better with `ShuffledHashJoin`, so for best performance it is often preferable to configure Comet to convert `SortMergeJoin` to `ShuffledHashJoin`. Comet does not yet provide spill-to-disk for `ShuffledHashJoin` so this could result in OOM. Also, `SortMergeJoin` may still be faster in some cases. It is best to test with both for your specific workloads. To configure Comet to convert `SortMergeJoin` to `ShuffledHashJoin`, set `spark.comet.exec.replaceSortMergeJoin=true`. ## Shuffle Comet provides accelerated shuffle implementations that can be used to improve the performance of your queries. To enable Comet shuffle, set the following configuration in your Spark configuration: ``` spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager spark.comet.exec.shuffle.enabled=true ``` `spark.shuffle.manager` is a Spark static configuration which cannot be changed at runtime. It must be set before the Spark context is created. You can enable or disable Comet shuffle at runtime by setting `spark.comet.exec.shuffle.enabled` to `true` or `false`. Once it is disabled, Comet will fall back to the default Spark shuffle manager. ### Shuffle Implementations Comet provides two shuffle implementations: Native Shuffle and Columnar Shuffle. Comet will first try to use Native Shuffle and if that is not possible it will try to use Columnar Shuffle. If neither can be applied, it will fall back to Spark for shuffle operations. #### Native Shuffle Comet provides a fully native shuffle implementation, which generally provides the best performance. Native shuffle supports `HashPartitioning`, `RangePartitioning` and `SinglePartitioning` but currently only supports primitive type partitioning keys. Columns that are not partitioning keys may contain complex types like maps, structs, and arrays. #### Columnar (JVM) Shuffle Comet Columnar shuffle is JVM-based and supports `HashPartitioning`, `RoundRobinPartitioning`, `RangePartitioning`, and `SinglePartitioning`. This shuffle implementation supports complex data types as partitioning keys. ### Shuffle Compression By default, Spark compresses shuffle files using LZ4 compression. Comet overrides this behavior with ZSTD compression. Compression can be disabled by setting `spark.shuffle.compress=false`, which may result in faster shuffle times in certain environments, such as single-node setups with fast NVMe drives, at the expense of increased disk space usage. ## Explain Plan ### Extended Explain With Spark 4.0.0 and newer, Comet can provide extended explain plan information in the Spark UI. Currently this lists reasons why Comet may not have been enabled for specific operations. To enable this, in the Spark configuration, set the following: ```shell -c spark.sql.extendedExplainProviders=org.apache.comet.ExtendedExplainInfo ``` This will add a section to the detailed plan displayed in the Spark SQL UI page. ================================================ FILE: docs/spark_expressions_support.md ================================================ # Supported Spark Expressions ### agg_funcs - [x] any - [x] any_value - [ ] approx_count_distinct - [ ] approx_percentile - [ ] array_agg - [x] avg - [x] bit_and - [x] bit_or - [x] bit_xor - [x] bool_and - [x] bool_or - [ ] collect_list - [ ] collect_set - [ ] corr - [x] count - [x] count_if - [ ] count_min_sketch - [x] covar_pop - [x] covar_samp - [x] every - [x] first - [x] first_value - [ ] grouping - [ ] grouping_id - [ ] histogram_numeric - [ ] kurtosis - [x] last - [x] last_value - [x] max - [ ] max_by - [x] mean - [ ] median - [x] min - [ ] min_by - [ ] mode - [ ] percentile - [ ] percentile_approx - [x] regr_avgx - [x] regr_avgy - [x] regr_count - [ ] regr_intercept - [ ] regr_r2 - [ ] regr_slope - [ ] regr_sxx - [ ] regr_sxy - [ ] regr_syy - [ ] skewness - [x] some - [x] std - [x] stddev - [x] stddev_pop - [x] stddev_samp - [x] sum - [ ] try_avg - [ ] try_sum - [x] var_pop - [x] var_samp - [x] variance ### array_funcs - [x] array - [x] array_append - [x] array_compact - [x] array_contains - [x] array_distinct - [x] array_except - [x] array_insert - [x] array_intersect - [x] array_join - [x] array_max - [ ] array_min - [ ] array_position - [x] array_remove - [x] array_repeat - [x] array_union - [x] arrays_overlap - [ ] arrays_zip - [x] element_at - [ ] flatten - [x] get - [ ] sequence - [ ] shuffle - [ ] slice - [x] sort_array ### bitwise_funcs - [x] & - [x] ^ - [ ] bit_count - [ ] bit_get - [ ] getbit - [x] shiftright - [ ] shiftrightunsigned - [x] | - [x] ~ ### collection_funcs - [ ] array_size - [ ] cardinality - [ ] concat - [x] reverse - [ ] size ### conditional_funcs - [x] coalesce - [x] if - [x] ifnull - [ ] nanvl - [x] nullif - [x] nvl - [x] nvl2 - [ ] when ### conversion_funcs - [ ] bigint - [ ] binary - [ ] boolean - [ ] cast - [ ] date - [ ] decimal - [ ] double - [ ] float - [ ] int - [ ] smallint - [ ] string - [ ] timestamp - [ ] tinyint ### csv_funcs - [ ] from_csv - [ ] schema_of_csv - [ ] to_csv ### datetime_funcs - [ ] add_months - [ ] convert_timezone - [x] curdate - [x] current_date - [ ] current_timestamp - [x] current_timezone - [ ] date_add - [ ] date_diff - [ ] date_format - [x] date_from_unix_date - [x] date_part - [ ] date_sub - [ ] date_trunc - [ ] dateadd - [ ] datediff - [x] datepart - [ ] day - [ ] dayofmonth - [ ] dayofweek - [ ] dayofyear - [x] extract - [x] from_unixtime - [ ] from_utc_timestamp - [ ] hour - [ ] last_day - [ ] localtimestamp - [ ] make_date - [ ] make_dt_interval - [ ] make_interval - [ ] make_timestamp - [ ] make_timestamp_ltz - [ ] make_timestamp_ntz - [ ] make_ym_interval - [ ] minute - [ ] month - [ ] months_between - [ ] next_day - [ ] now - [ ] quarter - [ ] second - [ ] timestamp_micros - [ ] timestamp_millis - [ ] timestamp_seconds - [ ] to_date - [ ] to_timestamp - [ ] to_timestamp_ltz - [ ] to_timestamp_ntz - [ ] to_unix_timestamp - [ ] to_utc_timestamp - [ ] trunc - [ ] try_to_timestamp - [ ] unix_date - [ ] unix_micros - [ ] unix_millis - [ ] unix_seconds - [x] unix_timestamp - [ ] weekday - [ ] weekofyear - [ ] year ### generator_funcs - [ ] explode - [ ] explode_outer - [ ] inline - [ ] inline_outer - [ ] posexplode - [ ] posexplode_outer - [ ] stack ### hash_funcs - [x] crc32 - [ ] hash - [x] md5 - [ ] sha - [ ] sha1 - [ ] sha2 - [ ] xxhash64 ### json_funcs - [ ] from_json - [x] get_json_object - [ ] json_array_length - [ ] json_object_keys - [ ] json_tuple - [ ] schema_of_json - [ ] to_json ### lambda_funcs - [ ] aggregate - [ ] array_sort - [ ] exists - [ ] filter - [ ] forall - [ ] map_filter - [ ] map_zip_with - [ ] reduce - [ ] transform - [ ] transform_keys - [ ] transform_values - [ ] zip_with ### map_funcs - [ ] element_at - [ ] map - [ ] map_concat - [x] map_contains_key - [ ] map_entries - [ ] map_from_arrays - [ ] map_from_entries - [x] map_keys - [ ] map_values - [ ] str_to_map - [ ] try_element_at ### math_funcs - [x] % - [x] - - [x] - - [x] - - [x] / - [x] abs - [x] acos - [ ] acosh - [x] asin - [ ] asinh - [x] atan - [x] atan2 - [ ] atanh - [x] bin - [ ] bround - [ ] cbrt - [x] ceil - [x] ceiling - [ ] conv - [x] cos - [ ] cosh - [ ] cot - [ ] csc - [ ] degrees - [ ] div - [ ] e - [x] exp - [ ] expm1 - [ ] factorial - [x] floor - [ ] greatest - [ ] hex - [ ] hypot - [ ] least - [x] ln - [ ] log - [x] log10 - [ ] log1p - [x] log2 - [x] mod - [x] negative - [ ] pi - [ ] pmod - [x] positive - [x] pow - [x] power - [ ] radians - [ ] rand - [ ] randn - [ ] random - [ ] rint - [x] round - [ ] sec - [x] shiftleft - [x] sign - [x] signum - [x] sin - [ ] sinh - [x] sqrt - [x] tan - [ ] tanh - [x] try_add - [x] try_divide - [x] try_multiply - [x] try_subtract - [x] unhex - [x] width_bucket ### misc_funcs - [ ] aes_decrypt - [ ] aes_encrypt - [ ] assert_true - [x] current_catalog - [x] current_database - [x] current_schema - [x] current_user - [x] equal_null - [ ] input_file_block_length - [ ] input_file_block_start - [ ] input_file_name - [x] monotonically_increasing_id - [ ] raise_error - [x] rand - [x] randn - [x] spark_partition_id - [ ] typeof - [x] user - [ ] uuid - [ ] version ### predicate_funcs - [x] ! - [x] < - [x] <= - [x] <=> - [x] = - [x] == - [x] > - [x] > = - [x] and - [x] ilike - [x] in - [ ] isnan - [x] isnotnull - [x] isnull - [x] like - [x] not - [x] or - [ ] regexp - [ ] regexp_like - [ ] rlike ### string_funcs - [x] ascii - [ ] base64 - [x] bit_length - [x] btrim - [x] char - [x] char_length - [x] character_length - [x] chr - [x] concat_ws - [x] contains - [ ] decode - [ ] elt - [ ] encode - [x] endswith - [ ] find_in_set - [ ] format_number - [ ] format_string - [x] initcap - [x] instr - [x] lcase - [ ] left - [x] len - [x] length - [ ] levenshtein - [ ] locate - [x] lower - [x] lpad - [x] ltrim - [ ] mask - [x] octet_length - [ ] overlay - [ ] position - [ ] printf - [ ] regexp_count - [ ] regexp_extract - [ ] regexp_extract_all - [ ] regexp_instr - [ ] regexp_replace - [ ] regexp_substr - [x] repeat - [x] replace - [ ] right - [x] rpad - [x] rtrim - [ ] sentences - [ ] soundex - [x] space - [ ] split - [ ] split_part - [x] startswith - [ ] substr - [ ] substring - [ ] substring_index - [ ] to_binary - [ ] to_char - [ ] to_number - [x] translate - [x] trim - [ ] try_to_binary - [ ] try_to_number - [x] ucase - [ ] unbase64 - [x] upper ### struct_funcs - [ ] named_struct - [ ] struct ### url_funcs - [ ] parse_url - [ ] url_decode - [ ] url_encode ### window_funcs - [ ] cume_dist - [ ] dense_rank - [ ] lag - [ ] lead - [ ] nth_value - [ ] ntile - [ ] percent_rank - [ ] rank - [ ] row_number ### xml_funcs - [ ] xpath - [ ] xpath_boolean - [ ] xpath_double - [ ] xpath_float - [ ] xpath_int - [ ] xpath_long - [ ] xpath_number - [ ] xpath_short - [ ] xpath_string ================================================ FILE: fuzz-testing/.gitignore ================================================ .idea target spark-warehouse queries.sql results*.md test*.parquet ================================================ FILE: fuzz-testing/README.md ================================================ # Comet Fuzz Comet Fuzz is a standalone project for generating random data and queries and executing queries against Spark with Comet disabled and enabled and checking for incompatibilities. Although it is a simple tool it has already been useful in finding many bugs. Comet Fuzz is inspired by the [SparkFuzz](https://ir.cwi.nl/pub/30222) paper from Databricks and CWI. ## Roadmap Planned areas of improvement: - ANSI mode - Support for all data types, expressions, and operators supported by Comet - IF and CASE WHEN expressions - Complex (nested) expressions - Literal scalar values in queries - Add option to avoid grouping and sorting on floating-point columns - Improve join query support: - Support joins without join keys - Support composite join keys - Support multiple join keys - Support join conditions that use expressions ## Usage From the root of the project, run `mvn install -DskipTests` to install Comet. Then build the fuzz testing jar. ```shell mvn package ``` Set appropriate values for `SPARK_HOME`, `SPARK_MASTER`, and `COMET_JAR` environment variables and then use `spark-submit` to run CometFuzz against a Spark cluster. ### Generating Data Files ```shell $SPARK_HOME/bin/spark-submit \ --master $SPARK_MASTER \ --class org.apache.comet.fuzz.Main \ target/comet-fuzz-spark3.5_2.12-0.13.0-SNAPSHOT-jar-with-dependencies.jar \ data --num-files=2 --num-rows=200 --exclude-negative-zero --generate-arrays --generate-structs --generate-maps ``` There is an optional `--exclude-negative-zero` flag for excluding `-0.0` from the generated data, which is sometimes useful because we already know that we often have different behavior for this edge case due to differences between Rust and Java handling of this value. ### Generating Queries Generate random queries that are based on the available test files. ```shell $SPARK_HOME/bin/spark-submit \ --master $SPARK_MASTER \ --class org.apache.comet.fuzz.Main \ target/comet-fuzz-spark3.5_2.12-0.13.0-SNAPSHOT-jar-with-dependencies.jar \ queries --num-files=2 --num-queries=500 ``` Note that the output filename is currently hard-coded as `queries.sql` ### Execute Queries ```shell $SPARK_HOME/bin/spark-submit \ --master $SPARK_MASTER \ --conf spark.memory.offHeap.enabled=true \ --conf spark.memory.offHeap.size=16G \ --conf spark.plugins=org.apache.spark.CometPlugin \ --conf spark.comet.enabled=true \ --conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \ --conf spark.comet.exec.shuffle.enabled=true \ --jars $COMET_JAR \ --conf spark.driver.extraClassPath=$COMET_JAR \ --conf spark.executor.extraClassPath=$COMET_JAR \ --class org.apache.comet.fuzz.Main \ target/comet-fuzz-spark3.5_2.12-0.13.0-SNAPSHOT-jar-with-dependencies.jar \ run --num-files=2 --filename=queries.sql ``` Note that the output filename is currently hard-coded as `results-${System.currentTimeMillis()}.md` ### Compare existing datasets To compare a pair of existing datasets you can use a comparison tool. The example below is for TPC-H queries results generated by pure Spark and Comet ```shell $SPARK_HOME/bin/spark-submit \ --master $SPARK_MASTER \ --class org.apache.comet.fuzz.ComparisonTool target/comet-fuzz-spark3.5_2.12-0.13.0-SNAPSHOT-jar-with-dependencies.jar \ compareParquet --input-spark-folder=/tmp/tpch/spark --input-comet-folder=/tmp/tpch/comet ``` The tool takes a pair of existing folders of the same layout and compares subfolders treating them as parquet based datasets ================================================ FILE: fuzz-testing/pom.xml ================================================ 4.0.0 org.apache.datafusion comet-parent-spark${spark.version.short}_${scala.binary.version} 0.15.0-SNAPSHOT ../pom.xml comet-fuzz-spark${spark.version.short}_${scala.binary.version} comet-fuzz http://maven.apache.org jar false org.scala-lang scala-library ${scala.version} provided org.apache.spark spark-sql_${scala.binary.version} provided org.scala-lang.modules scala-collection-compat_${scala.binary.version} org.apache.datafusion comet-spark-spark${spark.version.short}_${scala.binary.version} ${project.version} org.scala-lang.modules scala-collection-compat_${scala.binary.version} org.rogach scallop_${scala.binary.version} src/main/scala org.apache.maven.plugins maven-compiler-plugin ${maven-compiler-plugin.version} ${java.version} ${java.version} net.alchim31.maven scala-maven-plugin ${scala.plugin.version} compile testCompile maven-assembly-plugin ${maven-assembly-plugin.version} jar-with-dependencies make-assembly package single org.apache.maven.plugins maven-install-plugin true ================================================ FILE: fuzz-testing/run.sh ================================================ #!/bin/bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # # Usage: ./run.sh # Builds necessary JARs, generates data and queries, and runs fuzz tests for Comet Spark. # Environment variables: # SPARK_HOME - path to Spark installation # SPARK_MASTER - Spark master URL (default: local[*]) # SCALA_MAJOR_VERSION - Scala major version to use (default: 2.12) # SPARK_MAJOR_VERSION - Spark major version to use (default: 3.5) # NUM_FILES - number of data files to generate (default: 2) # NUM_ROWS - number of rows per file (default: 200) # NUM_QUERIES - number of queries to generate (default: 500) set -eux DIR="$(cd "$(dirname "$0")" && pwd)" PARENT_DIR="${DIR}/.." MVN_CMD="${PARENT_DIR}/mvnw" SPARK_MASTER="${SPARK_MASTER:-local[*]}" SCALA_MAJOR_VERSION="${SCALA_MAJOR_VERSION:-2.12}" SPARK_MAJOR_VERSION="${SPARK_MAJOR_VERSION:-3.5}" PROFILES="-Pscala-${SCALA_MAJOR_VERSION},spark-${SPARK_MAJOR_VERSION}" PROJECT_VERSION=$("${MVN_CMD}" -f "${DIR}/pom.xml" -q help:evaluate -Dexpression=project.version -DforceStdout) COMET_SPARK_JAR="${PARENT_DIR}/spark/target/comet-spark${SPARK_MAJOR_VERSION}_${SCALA_MAJOR_VERSION}-${PROJECT_VERSION}.jar" COMET_FUZZ_JAR="${DIR}/target/comet-fuzz-spark${SPARK_MAJOR_VERSION}_${SCALA_MAJOR_VERSION}-${PROJECT_VERSION}-jar-with-dependencies.jar" NUM_FILES="${NUM_FILES:-2}" NUM_ROWS="${NUM_ROWS:-200}" NUM_QUERIES="${NUM_QUERIES:-500}" if [ ! -f "${COMET_SPARK_JAR}" ]; then echo "Building Comet Spark jar..." pushd "${PARENT_DIR}" PROFILES="${PROFILES}" make popd else echo "Building Fuzz testing jar..." "${MVN_CMD}" -f "${DIR}/pom.xml" package -DskipTests "${PROFILES}" fi echo "Generating data..." "${SPARK_HOME}/bin/spark-submit" \ --master "${SPARK_MASTER}" \ --class org.apache.comet.fuzz.Main \ "${COMET_FUZZ_JAR}" \ data --num-files="${NUM_FILES}" --num-rows="${NUM_ROWS}" \ --exclude-negative-zero \ --generate-arrays --generate-structs --generate-maps echo "Generating queries..." "${SPARK_HOME}/bin/spark-submit" \ --master "${SPARK_MASTER}" \ --class org.apache.comet.fuzz.Main \ "${COMET_FUZZ_JAR}" \ queries --num-files="${NUM_FILES}" --num-queries="${NUM_QUERIES}" echo "Running fuzz tests..." "${SPARK_HOME}/bin/spark-submit" \ --master "${SPARK_MASTER}" \ --conf spark.memory.offHeap.enabled=true \ --conf spark.memory.offHeap.size=16G \ --conf spark.plugins=org.apache.spark.CometPlugin \ --conf spark.comet.enabled=true \ --conf spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager \ --conf spark.comet.exec.shuffle.enabled=true \ --jars "${COMET_SPARK_JAR}" \ --conf spark.driver.extraClassPath="${COMET_SPARK_JAR}" \ --conf spark.executor.extraClassPath="${COMET_SPARK_JAR}" \ --class org.apache.comet.fuzz.Main \ "${COMET_FUZZ_JAR}" \ run --num-files="${NUM_FILES}" --filename="queries.sql" ================================================ FILE: fuzz-testing/src/main/scala/org/apache/comet/fuzz/ComparisonTool.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.fuzz import java.io.File import org.rogach.scallop.{ScallopConf, ScallopOption, Subcommand} import org.apache.spark.sql.{functions, SparkSession} class ComparisonToolConf(arguments: Seq[String]) extends ScallopConf(arguments) { object compareParquet extends Subcommand("compareParquet") { val inputSparkFolder: ScallopOption[String] = opt[String](required = true, descr = "Folder with Spark produced results in Parquet format") val inputCometFolder: ScallopOption[String] = opt[String](required = true, descr = "Folder with Comet produced results in Parquet format") val tolerance: ScallopOption[Double] = opt[Double](default = Some(0.000002), descr = "Tolerance for floating point comparisons") } addSubcommand(compareParquet) verify() } object ComparisonTool { lazy val spark: SparkSession = SparkSession .builder() .getOrCreate() def main(args: Array[String]): Unit = { val conf = new ComparisonToolConf(args.toIndexedSeq) conf.subcommand match { case Some(conf.compareParquet) => compareParquetFolders( spark, conf.compareParquet.inputSparkFolder(), conf.compareParquet.inputCometFolder(), conf.compareParquet.tolerance()) case _ => // scalastyle:off println println("Invalid subcommand") // scalastyle:on println sys.exit(-1) } } private def compareParquetFolders( spark: SparkSession, sparkFolderPath: String, cometFolderPath: String, tolerance: Double): Unit = { val output = QueryRunner.createOutputMdFile() try { val sparkFolder = new File(sparkFolderPath) val cometFolder = new File(cometFolderPath) if (!sparkFolder.exists() || !sparkFolder.isDirectory) { throw new IllegalArgumentException( s"Spark folder does not exist or is not a directory: $sparkFolderPath") } if (!cometFolder.exists() || !cometFolder.isDirectory) { throw new IllegalArgumentException( s"Comet folder does not exist or is not a directory: $cometFolderPath") } // Get all subdirectories from the Spark folder val sparkSubfolders = sparkFolder .listFiles() .filter(_.isDirectory) .map(_.getName) .sorted output.write("# Comparing Parquet Folders\n\n") output.write(s"Spark folder: $sparkFolderPath\n") output.write(s"Comet folder: $cometFolderPath\n") output.write(s"Found ${sparkSubfolders.length} subfolders to compare\n\n") // Compare each subfolder sparkSubfolders.foreach { subfolderName => val sparkSubfolderPath = new File(sparkFolder, subfolderName) val cometSubfolderPath = new File(cometFolder, subfolderName) if (!cometSubfolderPath.exists() || !cometSubfolderPath.isDirectory) { output.write(s"## Subfolder: $subfolderName\n") output.write( s"[WARNING] Comet subfolder not found: ${cometSubfolderPath.getAbsolutePath}\n\n") } else { output.write(s"## Comparing subfolder: $subfolderName\n\n") try { // Read Spark parquet files spark.conf.set("spark.comet.enabled", "false") val sparkDf = spark.read.parquet(sparkSubfolderPath.getAbsolutePath) val sparkRows = sparkDf.orderBy(sparkDf.columns.map(functions.col): _*).collect() // Read Comet parquet files val cometDf = spark.read.parquet(cometSubfolderPath.getAbsolutePath) val cometRows = cometDf.orderBy(cometDf.columns.map(functions.col): _*).collect() // Compare the results if (QueryComparison.assertSameRows(sparkRows, cometRows, output, tolerance)) { output.write(s"Subfolder $subfolderName: ${sparkRows.length} rows matched\n\n") } else { // Output schema if dataframes are not equal QueryComparison.showSchema( output, sparkDf.schema.treeString, cometDf.schema.treeString) } } catch { case e: Exception => output.write( s"[ERROR] Failed to compare subfolder $subfolderName: ${e.getMessage}\n") val sw = new java.io.StringWriter() val p = new java.io.PrintWriter(sw) e.printStackTrace(p) p.close() output.write(s"```\n${sw.toString}\n```\n\n") } } output.flush() } output.write("\n# Comparison Complete\n") output.write(s"Compared ${sparkSubfolders.length} subfolders\n") } finally { output.close() } } } ================================================ FILE: fuzz-testing/src/main/scala/org/apache/comet/fuzz/Main.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.fuzz import scala.util.Random import org.rogach.scallop.{ScallopConf, Subcommand} import org.rogach.scallop.ScallopOption import org.apache.spark.sql.SparkSession import org.apache.comet.testing.{DataGenOptions, ParquetGenerator, SchemaGenOptions} class Conf(arguments: Seq[String]) extends ScallopConf(arguments) { object generateData extends Subcommand("data") { val numFiles: ScallopOption[Int] = opt[Int](required = true, descr = "Number of files to generate") val numRows: ScallopOption[Int] = opt[Int](required = true, descr = "Number of rows per file") val randomSeed: ScallopOption[Long] = opt[Long](required = false, descr = "Random seed to use") val generateArrays: ScallopOption[Boolean] = opt[Boolean](required = false, descr = "Whether to generate arrays") val generateStructs: ScallopOption[Boolean] = opt[Boolean](required = false, descr = "Whether to generate structs") val generateMaps: ScallopOption[Boolean] = opt[Boolean](required = false, descr = "Whether to generate maps") val excludeNegativeZero: ScallopOption[Boolean] = opt[Boolean](required = false, descr = "Whether to exclude negative zero") } addSubcommand(generateData) object generateQueries extends Subcommand("queries") { val numFiles: ScallopOption[Int] = opt[Int](required = true, descr = "Number of input files to use") val numQueries: ScallopOption[Int] = opt[Int](required = true, descr = "Number of queries to generate") val randomSeed: ScallopOption[Long] = opt[Long](required = false, descr = "Random seed to use") } addSubcommand(generateQueries) object runQueries extends Subcommand("run") { val filename: ScallopOption[String] = opt[String](required = true, descr = "File to write queries to") val numFiles: ScallopOption[Int] = opt[Int](required = true, descr = "Number of input files to use") val showFailedSparkQueries: ScallopOption[Boolean] = opt[Boolean](required = false, descr = "Whether to show failed Spark queries") } addSubcommand(runQueries) verify() } object Main { lazy val spark: SparkSession = SparkSession .builder() .getOrCreate() def main(args: Array[String]): Unit = { val conf = new Conf(args.toIndexedSeq) conf.subcommand match { case Some(conf.generateData) => val r = conf.generateData.randomSeed.toOption match { case Some(seed) => new Random(seed) case None => new Random() } for (i <- 0 until conf.generateData.numFiles()) { ParquetGenerator.makeParquetFile( r, spark, s"test$i.parquet", numRows = conf.generateData.numRows(), SchemaGenOptions( generateArray = conf.generateData.generateArrays(), generateStruct = conf.generateData.generateStructs(), generateMap = conf.generateData.generateMaps(), // create two columns of each primitive type so that they can be used in binary // expressions such as `a + b` and `a < b` primitiveTypes = SchemaGenOptions.defaultPrimitiveTypes ++ SchemaGenOptions.defaultPrimitiveTypes), DataGenOptions( allowNull = true, generateNegativeZero = !conf.generateData.excludeNegativeZero())) } case Some(conf.generateQueries) => val r = conf.generateQueries.randomSeed.toOption match { case Some(seed) => new Random(seed) case None => new Random() } QueryGen.generateRandomQueries( r, spark, numFiles = conf.generateQueries.numFiles(), conf.generateQueries.numQueries()) case Some(conf.runQueries) => QueryRunner.runQueries( spark, conf.runQueries.numFiles(), conf.runQueries.filename(), conf.runQueries.showFailedSparkQueries()) case _ => // scalastyle:off println println("Invalid subcommand") // scalastyle:on println sys.exit(-1) } } } ================================================ FILE: fuzz-testing/src/main/scala/org/apache/comet/fuzz/Meta.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.fuzz import org.apache.spark.sql.types.DataType import org.apache.spark.sql.types.DataTypes sealed trait SparkType case class SparkTypeOneOf(dataTypes: Seq[SparkType]) extends SparkType case object SparkBooleanType extends SparkType case object SparkBinaryType extends SparkType case object SparkStringType extends SparkType case object SparkIntegralType extends SparkType case object SparkByteType extends SparkType case object SparkShortType extends SparkType case object SparkIntType extends SparkType case object SparkLongType extends SparkType case object SparkFloatType extends SparkType case object SparkDoubleType extends SparkType case class SparkDecimalType(p: Int, s: Int) extends SparkType case object SparkNumericType extends SparkType case object SparkDateType extends SparkType case object SparkTimestampType extends SparkType case object SparkDateOrTimestampType extends SparkType case class SparkArrayType(elementType: SparkType) extends SparkType case class SparkMapType(keyType: SparkType, valueType: SparkType) extends SparkType case class SparkStructType(fields: Seq[SparkType]) extends SparkType case object SparkAnyType extends SparkType case class FunctionSignature(inputTypes: Seq[SparkType], varArgs: Boolean = false) case class Function(name: String, signatures: Seq[FunctionSignature]) object Meta { val primitiveSparkTypes: Seq[SparkType] = Seq( SparkBooleanType, SparkBinaryType, SparkStringType, SparkByteType, SparkShortType, SparkIntType, SparkLongType, SparkFloatType, SparkDoubleType, SparkDateType, SparkTimestampType) val dataTypes: Seq[(DataType, Double)] = Seq( (DataTypes.BooleanType, 0.1), (DataTypes.ByteType, 0.2), (DataTypes.ShortType, 0.2), (DataTypes.IntegerType, 0.2), (DataTypes.LongType, 0.2), (DataTypes.FloatType, 0.2), (DataTypes.DoubleType, 0.2), (DataTypes.createDecimalType(10, 2), 0.2), (DataTypes.DateType, 0.2), (DataTypes.TimestampType, 0.2), (DataTypes.TimestampNTZType, 0.2), (DataTypes.StringType, 0.2), (DataTypes.BinaryType, 0.1)) private def createFunctionWithInputTypes( name: String, inputs: Seq[SparkType], varArgs: Boolean = false): Function = { Function(name, Seq(FunctionSignature(inputs, varArgs))) createFunctions(name, Seq(FunctionSignature(inputs, varArgs))) } private def createFunctions(name: String, signatures: Seq[FunctionSignature]): Function = { signatures.foreach { s => assert( !s.varArgs || s.inputTypes.length == 1, s"Variadic function $s must have exactly one input type") } Function(name, signatures) } private def createUnaryStringFunction(name: String): Function = { createFunctionWithInputTypes(name, Seq(SparkStringType)) } private def createUnaryNumericFunction(name: String): Function = { createFunctionWithInputTypes(name, Seq(SparkNumericType)) } // Math expressions (corresponds to mathExpressions in QueryPlanSerde) val mathScalarFunc: Seq[Function] = Seq( createUnaryNumericFunction("abs"), createUnaryNumericFunction("acos"), createUnaryNumericFunction("asin"), createUnaryNumericFunction("atan"), createFunctionWithInputTypes("atan2", Seq(SparkNumericType, SparkNumericType)), createUnaryNumericFunction("cos"), createUnaryNumericFunction("cosh"), createUnaryNumericFunction("exp"), createUnaryNumericFunction("expm1"), createFunctionWithInputTypes("log", Seq(SparkNumericType, SparkNumericType)), createUnaryNumericFunction("log10"), createUnaryNumericFunction("log2"), createFunctionWithInputTypes("pow", Seq(SparkNumericType, SparkNumericType)), createFunctionWithInputTypes("remainder", Seq(SparkNumericType, SparkNumericType)), createFunctions( "round", Seq( FunctionSignature(Seq(SparkNumericType)), FunctionSignature(Seq(SparkNumericType, SparkIntType)))), createUnaryNumericFunction("signum"), createUnaryNumericFunction("sin"), createUnaryNumericFunction("sinh"), createUnaryNumericFunction("sqrt"), createUnaryNumericFunction("tan"), createUnaryNumericFunction("tanh"), createUnaryNumericFunction("cot"), createUnaryNumericFunction("ceil"), createUnaryNumericFunction("floor"), createFunctionWithInputTypes("unary_minus", Seq(SparkNumericType))) // Hash expressions (corresponds to hashExpressions in QueryPlanSerde) val hashScalarFunc: Seq[Function] = Seq( createFunctionWithInputTypes("md5", Seq(SparkAnyType)), createFunctionWithInputTypes("murmur3_hash", Seq(SparkAnyType)), // TODO variadic createFunctionWithInputTypes("sha2", Seq(SparkAnyType, SparkIntType))) // String expressions (corresponds to stringExpressions in QueryPlanSerde) val stringScalarFunc: Seq[Function] = Seq( createUnaryStringFunction("ascii"), createUnaryStringFunction("bit_length"), createUnaryStringFunction("chr"), createFunctionWithInputTypes( "concat", Seq( SparkTypeOneOf( Seq( SparkStringType, SparkBinaryType, SparkArrayType(SparkStringType), SparkArrayType(SparkNumericType), SparkArrayType(SparkBinaryType)))), varArgs = true), createFunctionWithInputTypes("concat_ws", Seq(SparkStringType, SparkStringType)), createFunctionWithInputTypes("contains", Seq(SparkStringType, SparkStringType)), createFunctionWithInputTypes("ends_with", Seq(SparkStringType, SparkStringType)), createFunctionWithInputTypes( "hex", Seq(SparkTypeOneOf(Seq(SparkStringType, SparkBinaryType, SparkIntType, SparkLongType)))), createUnaryStringFunction("init_cap"), createFunctionWithInputTypes("instr", Seq(SparkStringType, SparkStringType)), createFunctionWithInputTypes( "length", Seq(SparkTypeOneOf(Seq(SparkStringType, SparkBinaryType)))), createFunctionWithInputTypes("like", Seq(SparkStringType, SparkStringType)), createUnaryStringFunction("lower"), createFunctions( "lpad", Seq( FunctionSignature(Seq(SparkStringType, SparkIntegralType)), FunctionSignature(Seq(SparkStringType, SparkIntegralType, SparkStringType)))), createUnaryStringFunction("ltrim"), createUnaryStringFunction("octet_length"), createFunctions( "regexp_replace", Seq( FunctionSignature(Seq(SparkStringType, SparkStringType, SparkStringType)), FunctionSignature(Seq(SparkStringType, SparkStringType, SparkStringType, SparkIntType)))), createFunctionWithInputTypes("repeat", Seq(SparkStringType, SparkIntType)), createFunctions( "replace", Seq( FunctionSignature(Seq(SparkStringType, SparkStringType)), FunctionSignature(Seq(SparkStringType, SparkStringType, SparkStringType)))), createFunctions( "reverse", Seq( FunctionSignature(Seq(SparkStringType)), FunctionSignature(Seq(SparkArrayType(SparkAnyType))))), createFunctionWithInputTypes("rlike", Seq(SparkStringType, SparkStringType)), createFunctions( "rpad", Seq( FunctionSignature(Seq(SparkStringType, SparkIntegralType)), FunctionSignature(Seq(SparkStringType, SparkIntegralType, SparkStringType)))), createUnaryStringFunction("rtrim"), createFunctions( "split", Seq( FunctionSignature(Seq(SparkStringType, SparkStringType)), FunctionSignature(Seq(SparkStringType, SparkStringType, SparkIntType)))), createFunctionWithInputTypes("starts_with", Seq(SparkStringType, SparkStringType)), createFunctionWithInputTypes("string_space", Seq(SparkIntType)), createFunctionWithInputTypes("substring", Seq(SparkStringType, SparkIntType, SparkIntType)), createFunctionWithInputTypes("translate", Seq(SparkStringType, SparkStringType)), createUnaryStringFunction("trim"), createUnaryStringFunction("btrim"), createUnaryStringFunction("unhex"), createUnaryStringFunction("upper"), createFunctionWithInputTypes("xxhash64", Seq(SparkAnyType)), // TODO variadic createFunctionWithInputTypes("sha1", Seq(SparkAnyType))) // Conditional expressions (corresponds to conditionalExpressions in QueryPlanSerde) val conditionalScalarFunc: Seq[Function] = Seq( createFunctionWithInputTypes("if", Seq(SparkBooleanType, SparkAnyType, SparkAnyType))) // Map expressions (corresponds to mapExpressions in QueryPlanSerde) val mapScalarFunc: Seq[Function] = Seq( createFunctionWithInputTypes( "map_extract", Seq(SparkMapType(SparkAnyType, SparkAnyType), SparkAnyType)), createFunctionWithInputTypes("map_keys", Seq(SparkMapType(SparkAnyType, SparkAnyType))), createFunctionWithInputTypes("map_entries", Seq(SparkMapType(SparkAnyType, SparkAnyType))), createFunctionWithInputTypes("map_values", Seq(SparkMapType(SparkAnyType, SparkAnyType))), createFunctionWithInputTypes( "map_from_arrays", Seq(SparkArrayType(SparkAnyType), SparkArrayType(SparkAnyType))), createFunctionWithInputTypes( "map_from_entries", Seq( SparkArrayType( SparkStructType(Seq( SparkTypeOneOf(primitiveSparkTypes.filterNot(_ == SparkBinaryType)), SparkTypeOneOf(primitiveSparkTypes.filterNot(_ == SparkBinaryType)))))))) // Predicate expressions (corresponds to predicateExpressions in QueryPlanSerde) val predicateScalarFunc: Seq[Function] = Seq( createFunctionWithInputTypes("and", Seq(SparkBooleanType, SparkBooleanType)), createFunctionWithInputTypes("or", Seq(SparkBooleanType, SparkBooleanType)), createFunctionWithInputTypes("not", Seq(SparkBooleanType)), createFunctionWithInputTypes("in", Seq(SparkAnyType, SparkAnyType)) ) // TODO: variadic // Struct expressions (corresponds to structExpressions in QueryPlanSerde) val structScalarFunc: Seq[Function] = Seq( createFunctionWithInputTypes( "create_named_struct", Seq(SparkStringType, SparkAnyType) ), // TODO: variadic name/value pairs createFunctionWithInputTypes( "get_struct_field", Seq(SparkStructType(Seq(SparkAnyType)), SparkStringType))) // Bitwise expressions (corresponds to bitwiseExpressions in QueryPlanSerde) val bitwiseScalarFunc: Seq[Function] = Seq( createFunctionWithInputTypes("bitwise_and", Seq(SparkIntegralType, SparkIntegralType)), createFunctionWithInputTypes("bitwise_count", Seq(SparkIntegralType)), createFunctionWithInputTypes("bitwise_get", Seq(SparkIntegralType, SparkIntType)), createFunctionWithInputTypes("bitwise_or", Seq(SparkIntegralType, SparkIntegralType)), createFunctionWithInputTypes("bitwise_not", Seq(SparkIntegralType)), createFunctionWithInputTypes("bitwise_xor", Seq(SparkIntegralType, SparkIntegralType)), createFunctionWithInputTypes("shift_left", Seq(SparkIntegralType, SparkIntType)), createFunctionWithInputTypes("shift_right", Seq(SparkIntegralType, SparkIntType))) // Misc expressions (corresponds to miscExpressions in QueryPlanSerde) val miscScalarFunc: Seq[Function] = Seq( createFunctionWithInputTypes("isnan", Seq(SparkNumericType)), createFunctionWithInputTypes("isnull", Seq(SparkAnyType)), createFunctionWithInputTypes("isnotnull", Seq(SparkAnyType)), createFunctionWithInputTypes("coalesce", Seq(SparkAnyType, SparkAnyType)) ) // TODO: variadic // Array expressions (corresponds to arrayExpressions in QueryPlanSerde) val arrayScalarFunc: Seq[Function] = Seq( createFunctionWithInputTypes("array_append", Seq(SparkArrayType(SparkAnyType), SparkAnyType)), createFunctionWithInputTypes("array_compact", Seq(SparkArrayType(SparkAnyType))), createFunctionWithInputTypes( "array_contains", Seq(SparkArrayType(SparkAnyType), SparkAnyType)), createFunctionWithInputTypes("array_distinct", Seq(SparkArrayType(SparkAnyType))), createFunctionWithInputTypes( "array_except", Seq(SparkArrayType(SparkAnyType), SparkArrayType(SparkAnyType))), createFunctionWithInputTypes( "array_insert", Seq(SparkArrayType(SparkAnyType), SparkIntType, SparkAnyType)), createFunctionWithInputTypes( "array_intersect", Seq(SparkArrayType(SparkAnyType), SparkArrayType(SparkAnyType))), createFunctions( "array_join", Seq( FunctionSignature(Seq(SparkArrayType(SparkAnyType), SparkStringType)), FunctionSignature(Seq(SparkArrayType(SparkAnyType), SparkStringType, SparkStringType)))), createFunctionWithInputTypes("array_max", Seq(SparkArrayType(SparkAnyType))), createFunctionWithInputTypes("array_min", Seq(SparkArrayType(SparkAnyType))), createFunctionWithInputTypes("array_remove", Seq(SparkArrayType(SparkAnyType), SparkAnyType)), createFunctionWithInputTypes("array_repeat", Seq(SparkAnyType, SparkIntType)), createFunctionWithInputTypes( "arrays_overlap", Seq(SparkArrayType(SparkAnyType), SparkArrayType(SparkAnyType))), createFunctionWithInputTypes( "array_union", Seq(SparkArrayType(SparkAnyType), SparkArrayType(SparkAnyType))), createFunctionWithInputTypes("array", Seq(SparkAnyType, SparkAnyType)), // TODO: variadic createFunctionWithInputTypes( "element_at", Seq( SparkTypeOneOf( Seq(SparkArrayType(SparkAnyType), SparkMapType(SparkAnyType, SparkAnyType))), SparkAnyType)), createFunctionWithInputTypes("flatten", Seq(SparkArrayType(SparkArrayType(SparkAnyType)))), createFunctionWithInputTypes( "get_array_item", Seq(SparkArrayType(SparkAnyType), SparkIntType))) // Temporal expressions (corresponds to temporalExpressions in QueryPlanSerde) val temporalScalarFunc: Seq[Function] = Seq( createFunctionWithInputTypes("date_add", Seq(SparkDateType, SparkIntType)), createFunctionWithInputTypes("date_sub", Seq(SparkDateType, SparkIntType)), createFunctions( "from_unixtime", Seq( FunctionSignature(Seq(SparkLongType)), FunctionSignature(Seq(SparkLongType, SparkStringType)))), createFunctionWithInputTypes("hour", Seq(SparkDateOrTimestampType)), createFunctionWithInputTypes("minute", Seq(SparkDateOrTimestampType)), createFunctionWithInputTypes("second", Seq(SparkDateOrTimestampType)), createFunctionWithInputTypes("trunc", Seq(SparkDateOrTimestampType, SparkStringType)), createFunctionWithInputTypes("year", Seq(SparkDateOrTimestampType)), createFunctionWithInputTypes("month", Seq(SparkDateOrTimestampType)), createFunctionWithInputTypes("day", Seq(SparkDateOrTimestampType)), createFunctionWithInputTypes("dayofmonth", Seq(SparkDateOrTimestampType)), createFunctionWithInputTypes("dayofweek", Seq(SparkDateOrTimestampType)), createFunctionWithInputTypes("weekday", Seq(SparkDateOrTimestampType)), createFunctionWithInputTypes("dayofyear", Seq(SparkDateOrTimestampType)), createFunctionWithInputTypes("weekofyear", Seq(SparkDateOrTimestampType)), createFunctionWithInputTypes("quarter", Seq(SparkDateOrTimestampType))) // Combined in same order as exprSerdeMap in QueryPlanSerde val scalarFunc: Seq[Function] = mathScalarFunc ++ hashScalarFunc ++ stringScalarFunc ++ conditionalScalarFunc ++ mapScalarFunc ++ predicateScalarFunc ++ structScalarFunc ++ bitwiseScalarFunc ++ miscScalarFunc ++ arrayScalarFunc ++ temporalScalarFunc val aggFunc: Seq[Function] = Seq( createFunctionWithInputTypes("min", Seq(SparkAnyType)), createFunctionWithInputTypes("max", Seq(SparkAnyType)), createFunctionWithInputTypes("count", Seq(SparkAnyType)), createUnaryNumericFunction("avg"), createUnaryNumericFunction("sum"), // first/last are non-deterministic and known to be incompatible with Spark // createFunctionWithInputTypes("first", Seq(SparkAnyType)), // createFunctionWithInputTypes("last", Seq(SparkAnyType)), createUnaryNumericFunction("var_pop"), createUnaryNumericFunction("var_samp"), createFunctionWithInputTypes("covar_pop", Seq(SparkNumericType, SparkNumericType)), createFunctionWithInputTypes("covar_samp", Seq(SparkNumericType, SparkNumericType)), createUnaryNumericFunction("stddev_pop"), createUnaryNumericFunction("stddev_samp"), createFunctionWithInputTypes("corr", Seq(SparkNumericType, SparkNumericType)), createFunctionWithInputTypes("bit_and", Seq(SparkIntegralType)), createFunctionWithInputTypes("bit_or", Seq(SparkIntegralType)), createFunctionWithInputTypes("bit_xor", Seq(SparkIntegralType))) val unaryArithmeticOps: Seq[String] = Seq("+", "-") val binaryArithmeticOps: Seq[String] = Seq("+", "-", "*", "/", "%", "&", "|", "^", "<<", ">>", "div") val comparisonOps: Seq[String] = Seq("=", "<=>", ">", ">=", "<", "<=") // TODO make this more comprehensive val comparisonTypes: Seq[SparkType] = Seq( SparkStringType, SparkBinaryType, SparkNumericType, SparkDateType, SparkTimestampType, SparkArrayType(SparkTypeOneOf(Seq(SparkStringType, SparkNumericType, SparkDateType)))) } ================================================ FILE: fuzz-testing/src/main/scala/org/apache/comet/fuzz/QueryGen.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.fuzz import java.io.{BufferedWriter, FileWriter} import scala.annotation.tailrec import scala.collection.mutable import scala.util.Random import org.apache.spark.sql.{DataFrame, SparkSession} import org.apache.spark.sql.types._ object QueryGen { def generateRandomQueries( r: Random, spark: SparkSession, numFiles: Int, numQueries: Int): Unit = { for (i <- 0 until numFiles) { spark.read.parquet(s"test$i.parquet").createTempView(s"test$i") } val w = new BufferedWriter(new FileWriter("queries.sql")) val uniqueQueries = mutable.HashSet[String]() for (_ <- 0 until numQueries) { try { val sql = r.nextInt().abs % 8 match { case 0 => generateJoin(r, spark, numFiles) case 1 => generateAggregate(r, spark, numFiles) case 2 => generateScalar(r, spark, numFiles) case 3 => generateCast(r, spark, numFiles) case 4 => generateUnaryArithmetic(r, spark, numFiles) case 5 => generateBinaryArithmetic(r, spark, numFiles) case 6 => generateBinaryComparison(r, spark, numFiles) case _ => generateConditional(r, spark, numFiles) } if (!uniqueQueries.contains(sql)) { uniqueQueries += sql w.write(sql + "\n") } } catch { case e: Exception => // scalastyle:off println(s"Failed to generate query: ${e.getMessage}") } } w.close() } private def generateAggregate(r: Random, spark: SparkSession, numFiles: Int): String = { val tableName = s"test${r.nextInt(numFiles)}" val table = spark.table(tableName) val func = Utils.randomChoice(Meta.aggFunc, r) try { val signature = Utils.randomChoice(func.signatures, r) val args = signature.inputTypes.map(x => pickRandomColumn(r, table, x)) val groupingCols = Range(0, 2).map(_ => Utils.randomChoice(table.columns, r)) if (groupingCols.isEmpty) { s"SELECT ${args.mkString(", ")}, ${func.name}(${args.mkString(", ")}) AS x " + s"FROM $tableName " + s"ORDER BY ${args.mkString(", ")};" } else { s"SELECT ${groupingCols.mkString(", ")}, ${func.name}(${args.mkString(", ")}) " + s"FROM $tableName " + s"GROUP BY ${groupingCols.mkString(",")} " + s"ORDER BY ${groupingCols.mkString(", ")};" } } catch { case e: Exception => throw new IllegalStateException( s"Failed to generate SQL for aggregate function ${func.name}", e) } } private def generateScalar(r: Random, spark: SparkSession, numFiles: Int): String = { val tableName = s"test${r.nextInt(numFiles)}" val table = spark.table(tableName) val func = Utils.randomChoice(Meta.scalarFunc, r) try { val signature = Utils.randomChoice(func.signatures, r) val args = if (signature.varArgs) { pickRandomColumns(r, table, signature.inputTypes.head) } else { signature.inputTypes.map(x => pickRandomColumn(r, table, x)) } // Example SELECT c0, log(c0) as x FROM test0 s"SELECT ${args.mkString(", ")}, ${func.name}(${args.mkString(", ")}) AS x " + s"FROM $tableName " + s"ORDER BY ${args.mkString(", ")};" } catch { case e: Exception => throw new IllegalStateException( s"Failed to generate SQL for scalar function ${func.name}", e) } } @tailrec private def pickRandomColumns(r: Random, df: DataFrame, targetType: SparkType): Seq[String] = { targetType match { case SparkTypeOneOf(choices) => val chosenType = Utils.randomChoice(choices, r) pickRandomColumns(r, df, chosenType) case _ => var columns = Set.empty[String] for (_ <- 0 to r.nextInt(df.columns.length)) { columns += pickRandomColumn(r, df, targetType) } columns.toSeq } } private def pickRandomColumn(r: Random, df: DataFrame, targetType: SparkType): String = { targetType match { case SparkAnyType => Utils.randomChoice(df.schema.fields, r).name case SparkBooleanType => select(r, df, _.dataType == BooleanType) case SparkByteType => select(r, df, _.dataType == ByteType) case SparkShortType => select(r, df, _.dataType == ShortType) case SparkIntType => select(r, df, _.dataType == IntegerType) case SparkLongType => select(r, df, _.dataType == LongType) case SparkFloatType => select(r, df, _.dataType == FloatType) case SparkDoubleType => select(r, df, _.dataType == DoubleType) case SparkDecimalType(_, _) => select(r, df, _.dataType.isInstanceOf[DecimalType]) case SparkIntegralType => select( r, df, f => f.dataType == ByteType || f.dataType == ShortType || f.dataType == IntegerType || f.dataType == LongType) case SparkNumericType => select(r, df, f => isNumeric(f.dataType)) case SparkStringType => select(r, df, _.dataType == StringType) case SparkBinaryType => select(r, df, _.dataType == BinaryType) case SparkDateType => select(r, df, _.dataType == DateType) case SparkTimestampType => select(r, df, _.dataType == TimestampType) case SparkDateOrTimestampType => select(r, df, f => f.dataType == DateType || f.dataType == TimestampType) case SparkTypeOneOf(choices) => pickRandomColumn(r, df, Utils.randomChoice(choices, r)) case SparkArrayType(elementType) => select( r, df, _.dataType match { case ArrayType(x, _) if typeMatch(elementType, x) => true case _ => false }) case SparkMapType(keyType, valueType) => select( r, df, _.dataType match { case MapType(k, v, _) if typeMatch(keyType, k) && typeMatch(valueType, v) => true case _ => false }) case SparkStructType(fields) => select( r, df, _.dataType match { case StructType(structFields) if structFields.length == fields.length => true case _ => false }) case _ => throw new IllegalStateException(targetType.toString) } } def pickTwoRandomColumns(r: Random, df: DataFrame, targetType: SparkType): (String, String) = { val a = pickRandomColumn(r, df, targetType) val df2 = df.drop(a) val b = pickRandomColumn(r, df2, targetType) (a, b) } /** Select a random field that matches a predicate */ private def select(r: Random, df: DataFrame, predicate: StructField => Boolean): String = { val candidates = df.schema.fields.filter(predicate) if (candidates.isEmpty) { throw new IllegalStateException("Failed to find suitable column") } Utils.randomChoice(candidates, r).name } private def isNumeric(d: DataType): Boolean = { d match { case _: ByteType | _: ShortType | _: IntegerType | _: LongType | _: FloatType | _: DoubleType | _: DecimalType => true case _ => false } } private def typeMatch(s: SparkType, d: DataType): Boolean = { (s, d) match { case (SparkAnyType, _) => true case (SparkBooleanType, BooleanType) => true case (SparkByteType, ByteType) => true case (SparkShortType, ShortType) => true case (SparkIntType, IntegerType) => true case (SparkLongType, LongType) => true case (SparkFloatType, FloatType) => true case (SparkDoubleType, DoubleType) => true case (SparkDecimalType(_, _), _: DecimalType) => true case (SparkIntegralType, ByteType | ShortType | IntegerType | LongType) => true case (SparkNumericType, _) if isNumeric(d) => true case (SparkStringType, StringType) => true case (SparkBinaryType, BinaryType) => true case (SparkDateType, DateType) => true case (SparkTimestampType, TimestampType | TimestampNTZType) => true case (SparkDateOrTimestampType, DateType | TimestampType | TimestampNTZType) => true case (SparkArrayType(elementType), ArrayType(elementDataType, _)) => typeMatch(elementType, elementDataType) case (SparkMapType(keyType, valueType), MapType(keyDataType, valueDataType, _)) => typeMatch(keyType, keyDataType) && typeMatch(valueType, valueDataType) case (SparkStructType(fields), StructType(structFields)) => fields.length == structFields.length && fields.zip(structFields.map(_.dataType)).forall { case (sparkType, dataType) => typeMatch(sparkType, dataType) } case (SparkTypeOneOf(choices), _) => choices.exists(choice => typeMatch(choice, d)) case _ => false } } private def generateUnaryArithmetic(r: Random, spark: SparkSession, numFiles: Int): String = { val tableName = s"test${r.nextInt(numFiles)}" val table = spark.table(tableName) val op = Utils.randomChoice(Meta.unaryArithmeticOps, r) val a = pickRandomColumn(r, table, SparkNumericType) // Example SELECT a, -a FROM test0 s"SELECT $a, $op$a " + s"FROM $tableName " + s"ORDER BY $a;" } private def generateBinaryArithmetic(r: Random, spark: SparkSession, numFiles: Int): String = { val tableName = s"test${r.nextInt(numFiles)}" val table = spark.table(tableName) val op = Utils.randomChoice(Meta.binaryArithmeticOps, r) val (a, b) = pickTwoRandomColumns(r, table, SparkNumericType) // Example SELECT a, b, a+b FROM test0 s"SELECT $a, $b, $a $op $b " + s"FROM $tableName " + s"ORDER BY $a, $b;" } private def generateBinaryComparison(r: Random, spark: SparkSession, numFiles: Int): String = { val tableName = s"test${r.nextInt(numFiles)}" val table = spark.table(tableName) val op = Utils.randomChoice(Meta.comparisonOps, r) // pick two columns with the same type val opType = Utils.randomChoice(Meta.comparisonTypes, r) val (a, b) = pickTwoRandomColumns(r, table, opType) // Example SELECT a, b, a <=> b FROM test0 s"SELECT $a, $b, $a $op $b " + s"FROM $tableName " + s"ORDER BY $a, $b;" } private def generateConditional(r: Random, spark: SparkSession, numFiles: Int): String = { val tableName = s"test${r.nextInt(numFiles)}" val table = spark.table(tableName) val op = Utils.randomChoice(Meta.comparisonOps, r) // pick two columns with the same type val opType = Utils.randomChoice(Meta.comparisonTypes, r) val (a, b) = pickTwoRandomColumns(r, table, opType) // Example SELECT a, b, IF(a <=> b, 1, 2), CASE WHEN a <=> b THEN 1 ELSE 2 END FROM test0 s"SELECT $a, $b, $a $op $b, IF($a $op $b, 1, 2), CASE WHEN $a $op $b THEN 1 ELSE 2 END " + s"FROM $tableName " + s"ORDER BY $a, $b;" } private def generateCast(r: Random, spark: SparkSession, numFiles: Int): String = { val tableName = s"test${r.nextInt(numFiles)}" val table = spark.table(tableName) val toType = Utils.randomWeightedChoice(Meta.dataTypes, r).sql val arg = Utils.randomChoice(table.columns, r) // We test both `cast` and `try_cast` to cover LEGACY and TRY eval modes. It is not // recommended to run Comet Fuzz with ANSI enabled currently. // Example SELECT c0, cast(c0 as float), try_cast(c0 as float) FROM test0 s"SELECT $arg, cast($arg as $toType), try_cast($arg as $toType) " + s"FROM $tableName " + s"ORDER BY $arg;" } private def generateJoin(r: Random, spark: SparkSession, numFiles: Int): String = { val leftTableName = s"test${r.nextInt(numFiles)}" val rightTableName = s"test${r.nextInt(numFiles)}" val leftTable = spark.table(leftTableName) val rightTable = spark.table(rightTableName) val leftCol = Utils.randomChoice(leftTable.columns, r) val rightCol = Utils.randomChoice(rightTable.columns, r) val joinTypes = Seq(("INNER", 0.4), ("LEFT", 0.3), ("RIGHT", 0.3)) val joinType = Utils.randomWeightedChoice(joinTypes, r) val leftColProjection = leftTable.columns.map(c => s"l.$c").mkString(", ") val rightColProjection = rightTable.columns.map(c => s"r.$c").mkString(", ") "SELECT " + s"$leftColProjection, " + s"$rightColProjection " + s"FROM $leftTableName l " + s"$joinType JOIN $rightTableName r " + s"ON l.$leftCol = r.$rightCol " + "ORDER BY " + s"$leftColProjection, " + s"$rightColProjection;" } } ================================================ FILE: fuzz-testing/src/main/scala/org/apache/comet/fuzz/QueryRunner.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.fuzz import java.io.{BufferedWriter, FileWriter, PrintWriter, StringWriter} import scala.collection.mutable import scala.io.Source import org.apache.spark.sql.{Row, SparkSession} import org.apache.comet.fuzz.QueryComparison.showPlans object QueryRunner { def createOutputMdFile(): BufferedWriter = { val outputFilename = s"results-${System.currentTimeMillis()}.md" // scalastyle:off println println(s"Writing results to $outputFilename") // scalastyle:on println new BufferedWriter(new FileWriter(outputFilename)) } def runQueries( spark: SparkSession, numFiles: Int, filename: String, showFailedSparkQueries: Boolean = false): Unit = { var queryCount = 0 var invalidQueryCount = 0 var cometFailureCount = 0 var cometSuccessCount = 0 val w = createOutputMdFile() // register input files for (i <- 0 until numFiles) { val table = spark.read.parquet(s"test$i.parquet") val tableName = s"test$i" table.createTempView(tableName) w.write( s"Created table $tableName with schema:\n\t" + s"${table.schema.fields.map(f => s"${f.name}: ${f.dataType}").mkString("\n\t")}\n\n") } val querySource = Source.fromFile(filename) try { querySource .getLines() .foreach(sql => { queryCount += 1 try { // execute with Spark spark.conf.set("spark.comet.enabled", "false") val df = spark.sql(sql) val sparkRows = df.collect() val sparkPlan = df.queryExecution.executedPlan.toString // execute with Comet try { spark.conf.set("spark.comet.enabled", "true") val df = spark.sql(sql) val cometRows = df.collect() val cometPlan = df.queryExecution.executedPlan.toString var success = QueryComparison.assertSameRows(sparkRows, cometRows, output = w) // check that the plan contains Comet operators if (!cometPlan.contains("Comet")) { success = false w.write("[ERROR] Comet did not accelerate any part of the plan\n") } QueryComparison.showSQL(w, sql) if (success) { cometSuccessCount += 1 } else { // show plans for failed queries showPlans(w, sparkPlan, cometPlan) cometFailureCount += 1 } } catch { case e: Exception => // the query worked in Spark but failed in Comet, so this is likely a bug in Comet cometFailureCount += 1 QueryComparison.showSQL(w, sql) w.write("### Spark Plan\n") w.write(s"```\n$sparkPlan\n```\n") w.write(s"[ERROR] Query failed in Comet: ${e.getMessage}:\n") w.write("```\n") val sw = new StringWriter() val p = new PrintWriter(sw) e.printStackTrace(p) p.close() w.write(s"${sw.toString}\n") w.write("```\n") } // flush after every query so that results are saved in the event of the driver crashing w.flush() } catch { case e: Exception => // we expect many generated queries to be invalid invalidQueryCount += 1 if (showFailedSparkQueries) { QueryComparison.showSQL(w, sql) w.write(s"Query failed in Spark: ${e.getMessage}\n") } } }) w.write("# Summary\n") w.write( s"Total queries: $queryCount; Invalid queries: $invalidQueryCount; " + s"Comet failed: $cometFailureCount; Comet succeeded: $cometSuccessCount\n") } finally { w.close() querySource.close() } } } object QueryComparison { def assertSameRows( sparkRows: Array[Row], cometRows: Array[Row], output: BufferedWriter, tolerance: Double = 0.000001): Boolean = { if (sparkRows.length == cometRows.length) { var i = 0 while (i < sparkRows.length) { val l = sparkRows(i) val r = cometRows(i) // Check the schema is equal for first row only if (i == 0 && l.schema != r.schema) { output.write("[ERROR] Spark produced different schema than Comet.\n") return false } assert(l.length == r.length) for (j <- 0 until l.length) { if (!same(l(j), r(j), tolerance)) { output.write(s"First difference at row $i:\n") output.write("Spark: `" + formatRow(l) + "`\n") output.write("Comet: `" + formatRow(r) + "`\n") i = sparkRows.length return false } } i += 1 } } else { output.write( s"[ERROR] Spark produced ${sparkRows.length} rows and " + s"Comet produced ${cometRows.length} rows.\n") return false } true } private def same(l: Any, r: Any, tolerance: Double): Boolean = { if (l == null || r == null) { return l == null && r == null } (l, r) match { case (a: Float, b: Float) if a.isPosInfinity => b.isPosInfinity case (a: Float, b: Float) if a.isNegInfinity => b.isNegInfinity case (a: Float, b: Float) if a.isInfinity => b.isInfinity case (a: Float, b: Float) if a.isNaN => b.isNaN case (a: Float, b: Float) => (a - b).abs <= tolerance case (a: Double, b: Double) if a.isPosInfinity => b.isPosInfinity case (a: Double, b: Double) if a.isNegInfinity => b.isNegInfinity case (a: Double, b: Double) if a.isInfinity => b.isInfinity case (a: Double, b: Double) if a.isNaN => b.isNaN case (a: Double, b: Double) => (a - b).abs <= tolerance case (a: Array[_], b: Array[_]) => a.length == b.length && a.zip(b).forall(x => same(x._1, x._2, tolerance)) case (a: mutable.WrappedArray[_], b: mutable.WrappedArray[_]) => a.length == b.length && a.zip(b).forall(x => same(x._1, x._2, tolerance)) case (a: Row, b: Row) => val aa = a.toSeq val bb = b.toSeq aa.length == bb.length && aa.zip(bb).forall(x => same(x._1, x._2, tolerance)) case (a, b) => a == b } } private def format(value: Any): String = { value match { case null => "NULL" case v: mutable.WrappedArray[_] => s"[${v.map(format).mkString(",")}]" case v: Array[Byte] => s"[${v.mkString(",")}]" case r: Row => formatRow(r) case other => other.toString } } private def formatRow(row: Row): String = { row.toSeq.map(format).mkString(",") } def showSQL(w: BufferedWriter, sql: String, maxLength: Int = 120): Unit = { w.write("## SQL\n") w.write("```\n") val words = sql.split(" ") val currentLine = new StringBuilder for (word <- words) { if (currentLine.length + word.length + 1 > maxLength) { w.write(currentLine.toString.trim) w.write("\n") currentLine.setLength(0) } currentLine.append(word).append(" ") } if (currentLine.nonEmpty) { w.write(currentLine.toString.trim) w.write("\n") } w.write("```\n") } def showPlans(w: BufferedWriter, sparkPlan: String, cometPlan: String): Unit = { w.write("### Spark Plan\n") w.write(s"```\n$sparkPlan\n```\n") w.write("### Comet Plan\n") w.write(s"```\n$cometPlan\n```\n") } def showSchema(w: BufferedWriter, sparkSchema: String, cometSchema: String): Unit = { w.write("### Spark Schema\n") w.write(s"```\n$sparkSchema\n```\n") w.write("### Comet Schema\n") w.write(s"```\n$cometSchema\n```\n") } } ================================================ FILE: fuzz-testing/src/main/scala/org/apache/comet/fuzz/Utils.scala ================================================ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.comet.fuzz import scala.util.Random object Utils { def randomChoice[T](list: Seq[T], r: Random): T = { list(r.nextInt(list.length)) } def randomWeightedChoice[T](valuesWithWeights: Seq[(T, Double)], r: Random): T = { val totalWeight = valuesWithWeights.map(_._2).sum val randomValue = r.nextDouble() * totalWeight var cumulativeWeight = 0.0 for ((value, weight) <- valuesWithWeights) { cumulativeWeight += weight if (cumulativeWeight >= randomValue) { return value } } // If for some reason the loop doesn't return, return the last value valuesWithWeights.last._1 } } ================================================ FILE: kube/Dockerfile ================================================ # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # FROM apache/spark:3.5.8 AS builder USER root # Installing JDK11 as the image comes with JRE RUN apt update \ && apt install -y curl \ && apt install -y openjdk-11-jdk \ && apt clean RUN apt install -y gcc-10 g++-10 cpp-10 unzip ENV CC="gcc-10" ENV CXX="g++-10" RUN PB_REL="https://github.com/protocolbuffers/protobuf/releases" \ && curl -LO $PB_REL/download/v30.2/protoc-30.2-linux-x86_64.zip \ && unzip protoc-30.2-linux-x86_64.zip -d /root/.local ENV PATH="$PATH:/root/.local/bin" RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ENV PATH="/root/.cargo/bin:${PATH}" ENV RUSTFLAGS="-C debuginfo=line-tables-only -C incremental=false" ENV SPARK_VERSION=3.5 ENV SCALA_VERSION=2.12 # copy source files to Docker image RUN mkdir /comet WORKDIR /comet # build native code first so that this layer can be re-used # if only Scala code gets modified COPY native /comet/native RUN cd native && RUSTFLAGS="-Ctarget-cpu=native" cargo build --release # copy the rest of the project COPY .mvn /comet/.mvn COPY mvnw /comet/mvnw COPY common /comet/common COPY dev /comet/dev COPY docs /comet/docs COPY fuzz-testing /comet/fuzz-testing COPY spark /comet/spark COPY spark-integration /comet/spark-integration COPY scalafmt.conf /comet/scalafmt.conf COPY .scalafix.conf /comet/.scalafix.conf COPY Makefile /comet/Makefile COPY pom.xml /comet/pom.xml RUN mkdir -p /root/.m2 && \ echo 'centralcentralhttps://repo1.maven.org/maven2' > /root/.m2/settings.xml # Pick the JDK instead of JRE to compile Comet RUN cd /comet \ && JAVA_HOME=$(readlink -f $(which javac) | sed "s/\/bin\/javac//") make release-nogit PROFILES="-Pspark-$SPARK_VERSION -Pscala-$SCALA_VERSION" FROM apache/spark:3.5.8 ENV SPARK_VERSION=3.5 ENV SCALA_VERSION=2.12 USER root # note the use of a wildcard in the file name so that this works with both snapshot and final release versions COPY --from=builder /comet/spark/target/comet-spark-spark${SPARK_VERSION}_$SCALA_VERSION-*.jar $SPARK_HOME/jars ================================================ FILE: kube/local/hadoop.env ================================================ # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # CORE_CONF_fs_defaultFS=hdfs://namenode:9000 CORE_CONF_hadoop_http_staticuser_user=root CORE_CONF_hadoop_proxyuser_hue_hosts=* CORE_CONF_hadoop_proxyuser_hue_groups=* CORE_CONF_io_compression_codecs=org.apache.hadoop.io.compress.SnappyCodec CORE_CONF_hadoop_tmp_dir=/hadoop-data CORE_CONF_dfs_client_use_datanode_hostname=true CORE_CONF_dfs_datanode_use_datanode_hostname=true HDFS_CONF_dfs_webhdfs_enabled=true HDFS_CONF_dfs_permissions_enabled=false HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false HDFS_CONF_dfs_client_use_datanode_hostname=true HDFS_CONF_dfs_datanode_use_datanode_hostname=true ================================================ FILE: kube/local/hdfs-docker-compose.yml ================================================ # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # version: "3" services: namenode: image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8 container_name: namenode restart: always ports: - 9870:9870 - 9000:9000 volumes: - /tmp/hadoop/dfs/name:/hadoop/dfs/name environment: - CLUSTER_NAME=test env_file: - hadoop.env datanode1: image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8 container_name: datanode1 hostname: datanode1 restart: always ports: - 9866:9866 - 9864:9864 depends_on: - namenode volumes: - /tmp/hadoop/dfs/data1:/hadoop/dfs/data environment: SERVICE_PRECONDITION: "namenode:9870" env_file: - hadoop.env datanode2: image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8 container_name: datanode2 hostname: datanode2 restart: always ports: - 9867:9866 - 9865:9864 depends_on: - namenode volumes: - /tmp/hadoop/dfs/data2:/hadoop/dfs/data environment: SERVICE_PRECONDITION: "namenode:9870" env_file: - hadoop.env datanode3: image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8 container_name: datanode3 hostname: datanode3 restart: always ports: - 9868:9866 - 9863:9864 depends_on: - namenode volumes: - /tmp/hadoop/dfs/data3:/hadoop/dfs/data environment: SERVICE_PRECONDITION: "namenode:9870" env_file: - hadoop.env resourcemanager: image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8 container_name: resourcemanager restart: always environment: SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode1:9864 datanode2:9864 datanode3:9864 datanode1:9866 datanode2:9866 datanode3:9866" env_file: - hadoop.env nodemanager1: image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8 container_name: nodemanager restart: always environment: SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode1:9864 datanode2:9864 datanode3:9864 datanode1:9866 datanode2:9866 datanode3:9866 resourcemanager:8088" env_file: - hadoop.env ================================================ FILE: mvnw ================================================ #!/bin/sh # ---------------------------------------------------------------------------- # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # ---------------------------------------------------------------------------- # ---------------------------------------------------------------------------- # Apache Maven Wrapper startup batch script, version 3.2.0 # # Required ENV vars: # ------------------ # JAVA_HOME - location of a JDK home dir # # Optional ENV vars # ----------------- # MAVEN_OPTS - parameters passed to the Java VM when running Maven # e.g. to debug Maven itself, use # set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 # MAVEN_SKIP_RC - flag to disable loading of mavenrc files # ---------------------------------------------------------------------------- if [ -z "$MAVEN_SKIP_RC" ] ; then if [ -f /usr/local/etc/mavenrc ] ; then . /usr/local/etc/mavenrc fi if [ -f /etc/mavenrc ] ; then . /etc/mavenrc fi if [ -f "$HOME/.mavenrc" ] ; then . "$HOME/.mavenrc" fi fi # OS specific support. $var _must_ be set to either true or false. cygwin=false; darwin=false; mingw=false case "$(uname)" in CYGWIN*) cygwin=true ;; MINGW*) mingw=true;; Darwin*) darwin=true # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home # See https://developer.apple.com/library/mac/qa/qa1170/_index.html if [ -z "$JAVA_HOME" ]; then if [ -x "/usr/libexec/java_home" ]; then JAVA_HOME="$(/usr/libexec/java_home)"; export JAVA_HOME else JAVA_HOME="/Library/Java/Home"; export JAVA_HOME fi fi ;; esac if [ -z "$JAVA_HOME" ] ; then if [ -r /etc/gentoo-release ] ; then JAVA_HOME=$(java-config --jre-home) fi fi # For Cygwin, ensure paths are in UNIX format before anything is touched if $cygwin ; then [ -n "$JAVA_HOME" ] && JAVA_HOME=$(cygpath --unix "$JAVA_HOME") [ -n "$CLASSPATH" ] && CLASSPATH=$(cygpath --path --unix "$CLASSPATH") fi # For Mingw, ensure paths are in UNIX format before anything is touched if $mingw ; then [ -n "$JAVA_HOME" ] && [ -d "$JAVA_HOME" ] && JAVA_HOME="$(cd "$JAVA_HOME" || (echo "cannot cd into $JAVA_HOME."; exit 1); pwd)" fi if [ -z "$JAVA_HOME" ]; then javaExecutable="$(which javac)" if [ -n "$javaExecutable" ] && ! [ "$(expr "\"$javaExecutable\"" : '\([^ ]*\)')" = "no" ]; then # readlink(1) is not available as standard on Solaris 10. readLink=$(which readlink) if [ ! "$(expr "$readLink" : '\([^ ]*\)')" = "no" ]; then if $darwin ; then javaHome="$(dirname "\"$javaExecutable\"")" javaExecutable="$(cd "\"$javaHome\"" && pwd -P)/javac" else javaExecutable="$(readlink -f "\"$javaExecutable\"")" fi javaHome="$(dirname "\"$javaExecutable\"")" javaHome=$(expr "$javaHome" : '\(.*\)/bin') JAVA_HOME="$javaHome" export JAVA_HOME fi fi fi if [ -z "$JAVACMD" ] ; then if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then # IBM's JDK on AIX uses strange locations for the executables JAVACMD="$JAVA_HOME/jre/sh/java" else JAVACMD="$JAVA_HOME/bin/java" fi else JAVACMD="$(\unset -f command 2>/dev/null; \command -v java)" fi fi if [ ! -x "$JAVACMD" ] ; then echo "Error: JAVA_HOME is not defined correctly." >&2 echo " We cannot execute $JAVACMD" >&2 exit 1 fi if [ -z "$JAVA_HOME" ] ; then echo "Warning: JAVA_HOME environment variable is not set." fi # traverses directory structure from process work directory to filesystem root # first directory with .mvn subdirectory is considered project base directory find_maven_basedir() { if [ -z "$1" ] then echo "Path not specified to find_maven_basedir" return 1 fi basedir="$1" wdir="$1" while [ "$wdir" != '/' ] ; do if [ -d "$wdir"/.mvn ] ; then basedir=$wdir break fi # workaround for JBEAP-8937 (on Solaris 10/Sparc) if [ -d "${wdir}" ]; then wdir=$(cd "$wdir/.." || exit 1; pwd) fi # end of workaround done printf '%s' "$(cd "$basedir" || exit 1; pwd)" } # concatenates all lines of a file concat_lines() { if [ -f "$1" ]; then # Remove \r in case we run on Windows within Git Bash # and check out the repository with auto CRLF management # enabled. Otherwise, we may read lines that are delimited with # \r\n and produce $'-Xarg\r' rather than -Xarg due to word # splitting rules. tr -s '\r\n' ' ' < "$1" fi } log() { if [ "$MVNW_VERBOSE" = true ]; then printf '%s\n' "$1" fi } BASE_DIR=$(find_maven_basedir "$(dirname "$0")") if [ -z "$BASE_DIR" ]; then exit 1; fi MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}; export MAVEN_PROJECTBASEDIR log "$MAVEN_PROJECTBASEDIR" ########################################################################################## # Extension to allow automatically downloading the maven-wrapper.jar from Maven-central # This allows using the maven wrapper in projects that prohibit checking in binary data. ########################################################################################## wrapperJarPath="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" if [ -r "$wrapperJarPath" ]; then log "Found $wrapperJarPath" else log "Couldn't find $wrapperJarPath, downloading it ..." if [ -n "$MVNW_REPOURL" ]; then wrapperUrl="$MVNW_REPOURL/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" else wrapperUrl="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" fi while IFS="=" read -r key value; do # Remove '\r' from value to allow usage on windows as IFS does not consider '\r' as a separator ( considers space, tab, new line ('\n'), and custom '=' ) safeValue=$(echo "$value" | tr -d '\r') case "$key" in (wrapperUrl) wrapperUrl="$safeValue"; break ;; esac done < "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.properties" log "Downloading from: $wrapperUrl" if $cygwin; then wrapperJarPath=$(cygpath --path --windows "$wrapperJarPath") fi if command -v wget > /dev/null; then log "Found wget ... using wget" [ "$MVNW_VERBOSE" = true ] && QUIET="" || QUIET="--quiet" if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then wget $QUIET "$wrapperUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath" else wget $QUIET --http-user="$MVNW_USERNAME" --http-password="$MVNW_PASSWORD" "$wrapperUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath" fi elif command -v curl > /dev/null; then log "Found curl ... using curl" [ "$MVNW_VERBOSE" = true ] && QUIET="" || QUIET="--silent" if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then curl $QUIET -o "$wrapperJarPath" "$wrapperUrl" -f -L || rm -f "$wrapperJarPath" else curl $QUIET --user "$MVNW_USERNAME:$MVNW_PASSWORD" -o "$wrapperJarPath" "$wrapperUrl" -f -L || rm -f "$wrapperJarPath" fi else log "Falling back to using Java to download" javaSource="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/MavenWrapperDownloader.java" javaClass="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/MavenWrapperDownloader.class" # For Cygwin, switch paths to Windows format before running javac if $cygwin; then javaSource=$(cygpath --path --windows "$javaSource") javaClass=$(cygpath --path --windows "$javaClass") fi if [ -e "$javaSource" ]; then if [ ! -e "$javaClass" ]; then log " - Compiling MavenWrapperDownloader.java ..." ("$JAVA_HOME/bin/javac" "$javaSource") fi if [ -e "$javaClass" ]; then log " - Running MavenWrapperDownloader.java ..." ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$wrapperUrl" "$wrapperJarPath") || rm -f "$wrapperJarPath" fi fi fi fi ########################################################################################## # End of extension ########################################################################################## # If specified, validate the SHA-256 sum of the Maven wrapper jar file wrapperSha256Sum="" while IFS="=" read -r key value; do case "$key" in (wrapperSha256Sum) wrapperSha256Sum=$value; break ;; esac done < "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.properties" if [ -n "$wrapperSha256Sum" ]; then wrapperSha256Result=false if command -v sha256sum > /dev/null; then if echo "$wrapperSha256Sum $wrapperJarPath" | sha256sum -c > /dev/null 2>&1; then wrapperSha256Result=true fi elif command -v shasum > /dev/null; then if echo "$wrapperSha256Sum $wrapperJarPath" | shasum -a 256 -c > /dev/null 2>&1; then wrapperSha256Result=true fi else echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." echo "Please install either command, or disable validation by removing 'wrapperSha256Sum' from your maven-wrapper.properties." exit 1 fi if [ $wrapperSha256Result = false ]; then echo "Error: Failed to validate Maven wrapper SHA-256, your Maven wrapper might be compromised." >&2 echo "Investigate or delete $wrapperJarPath to attempt a clean download." >&2 echo "If you updated your Maven version, you need to update the specified wrapperSha256Sum property." >&2 exit 1 fi fi MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" # For Cygwin, switch paths to Windows format before running java if $cygwin; then [ -n "$JAVA_HOME" ] && JAVA_HOME=$(cygpath --path --windows "$JAVA_HOME") [ -n "$CLASSPATH" ] && CLASSPATH=$(cygpath --path --windows "$CLASSPATH") [ -n "$MAVEN_PROJECTBASEDIR" ] && MAVEN_PROJECTBASEDIR=$(cygpath --path --windows "$MAVEN_PROJECTBASEDIR") fi # Provide a "standardized" way to retrieve the CLI args that will # work with both Windows and non-Windows executions. MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $*" export MAVEN_CMD_LINE_ARGS WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain # shellcheck disable=SC2086 # safe args exec "$JAVACMD" \ $MAVEN_OPTS \ $MAVEN_DEBUG_OPTS \ -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" ================================================ FILE: mvnw.cmd ================================================ @REM ---------------------------------------------------------------------------- @REM Licensed to the Apache Software Foundation (ASF) under one @REM or more contributor license agreements. See the NOTICE file @REM distributed with this work for additional information @REM regarding copyright ownership. The ASF licenses this file @REM to you under the Apache License, Version 2.0 (the @REM "License"); you may not use this file except in compliance @REM with the License. You may obtain a copy of the License at @REM @REM http://www.apache.org/licenses/LICENSE-2.0 @REM @REM Unless required by applicable law or agreed to in writing, @REM software distributed under the License is distributed on an @REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @REM KIND, either express or implied. See the License for the @REM specific language governing permissions and limitations @REM under the License. @REM ---------------------------------------------------------------------------- @REM ---------------------------------------------------------------------------- @REM Apache Maven Wrapper startup batch script, version 3.2.0 @REM @REM Required ENV vars: @REM JAVA_HOME - location of a JDK home dir @REM @REM Optional ENV vars @REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands @REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending @REM MAVEN_OPTS - parameters passed to the Java VM when running Maven @REM e.g. to debug Maven itself, use @REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 @REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files @REM ---------------------------------------------------------------------------- @REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' @echo off @REM set title of command window title %0 @REM enable echoing by setting MAVEN_BATCH_ECHO to 'on' @if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% @REM set %HOME% to equivalent of $HOME if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") @REM Execute a user defined script before this one if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre @REM check for pre script, once with legacy .bat ending and once with .cmd ending if exist "%USERPROFILE%\mavenrc_pre.bat" call "%USERPROFILE%\mavenrc_pre.bat" %* if exist "%USERPROFILE%\mavenrc_pre.cmd" call "%USERPROFILE%\mavenrc_pre.cmd" %* :skipRcPre @setlocal set ERROR_CODE=0 @REM To isolate internal variables from possible post scripts, we use another setlocal @setlocal @REM ==== START VALIDATION ==== if not "%JAVA_HOME%" == "" goto OkJHome echo. echo Error: JAVA_HOME not found in your environment. >&2 echo Please set the JAVA_HOME variable in your environment to match the >&2 echo location of your Java installation. >&2 echo. goto error :OkJHome if exist "%JAVA_HOME%\bin\java.exe" goto init echo. echo Error: JAVA_HOME is set to an invalid directory. >&2 echo JAVA_HOME = "%JAVA_HOME%" >&2 echo Please set the JAVA_HOME variable in your environment to match the >&2 echo location of your Java installation. >&2 echo. goto error @REM ==== END VALIDATION ==== :init @REM Find the project base dir, i.e. the directory that contains the folder ".mvn". @REM Fallback to current working directory if not found. set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir set EXEC_DIR=%CD% set WDIR=%EXEC_DIR% :findBaseDir IF EXIST "%WDIR%"\.mvn goto baseDirFound cd .. IF "%WDIR%"=="%CD%" goto baseDirNotFound set WDIR=%CD% goto findBaseDir :baseDirFound set MAVEN_PROJECTBASEDIR=%WDIR% cd "%EXEC_DIR%" goto endDetectBaseDir :baseDirNotFound set MAVEN_PROJECTBASEDIR=%EXEC_DIR% cd "%EXEC_DIR%" :endDetectBaseDir IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig @setlocal EnableExtensions EnableDelayedExpansion for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a @endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% :endReadAdditionalConfig SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain set WRAPPER_URL="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( IF "%%A"=="wrapperUrl" SET WRAPPER_URL=%%B ) @REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central @REM This allows using the maven wrapper in projects that prohibit checking in binary data. if exist %WRAPPER_JAR% ( if "%MVNW_VERBOSE%" == "true" ( echo Found %WRAPPER_JAR% ) ) else ( if not "%MVNW_REPOURL%" == "" ( SET WRAPPER_URL="%MVNW_REPOURL%/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" ) if "%MVNW_VERBOSE%" == "true" ( echo Couldn't find %WRAPPER_JAR%, downloading it ... echo Downloading from: %WRAPPER_URL% ) powershell -Command "&{"^ "$webclient = new-object System.Net.WebClient;"^ "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^ "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^ "}"^ "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%WRAPPER_URL%', '%WRAPPER_JAR%')"^ "}" if "%MVNW_VERBOSE%" == "true" ( echo Finished downloading %WRAPPER_JAR% ) ) @REM End of extension @REM If specified, validate the SHA-256 sum of the Maven wrapper jar file SET WRAPPER_SHA_256_SUM="" FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( IF "%%A"=="wrapperSha256Sum" SET WRAPPER_SHA_256_SUM=%%B ) IF NOT %WRAPPER_SHA_256_SUM%=="" ( powershell -Command "&{"^ "$hash = (Get-FileHash \"%WRAPPER_JAR%\" -Algorithm SHA256).Hash.ToLower();"^ "If('%WRAPPER_SHA_256_SUM%' -ne $hash){"^ " Write-Output 'Error: Failed to validate Maven wrapper SHA-256, your Maven wrapper might be compromised.';"^ " Write-Output 'Investigate or delete %WRAPPER_JAR% to attempt a clean download.';"^ " Write-Output 'If you updated your Maven version, you need to update the specified wrapperSha256Sum property.';"^ " exit 1;"^ "}"^ "}" if ERRORLEVEL 1 goto error ) @REM Provide a "standardized" way to retrieve the CLI args that will @REM work with both Windows and non-Windows executions. set MAVEN_CMD_LINE_ARGS=%* %MAVEN_JAVA_EXE% ^ %JVM_CONFIG_MAVEN_PROPS% ^ %MAVEN_OPTS% ^ %MAVEN_DEBUG_OPTS% ^ -classpath %WRAPPER_JAR% ^ "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" ^ %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* if ERRORLEVEL 1 goto error goto end :error set ERROR_CODE=1 :end @endlocal & set ERROR_CODE=%ERROR_CODE% if not "%MAVEN_SKIP_RC%"=="" goto skipRcPost @REM check for post script, once with legacy .bat ending and once with .cmd ending if exist "%USERPROFILE%\mavenrc_post.bat" call "%USERPROFILE%\mavenrc_post.bat" if exist "%USERPROFILE%\mavenrc_post.cmd" call "%USERPROFILE%\mavenrc_post.cmd" :skipRcPost @REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' if "%MAVEN_BATCH_PAUSE%"=="on" pause if "%MAVEN_TERMINATE_CMD%"=="on" exit %ERROR_CODE% cmd /C exit /B %ERROR_CODE% ================================================ FILE: native/Cargo.toml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. [workspace] default-members = ["core", "spark-expr", "common", "proto", "jni-bridge", "shuffle"] members = ["core", "spark-expr", "common", "proto", "jni-bridge", "shuffle", "hdfs", "fs-hdfs"] resolver = "2" [workspace.package] version = "0.15.0" homepage = "https://datafusion.apache.org/comet" repository = "https://github.com/apache/datafusion-comet" authors = ["Apache DataFusion "] description = "Apache DataFusion Comet: High performance accelerator for Apache Spark" readme = "README.md" license = "Apache-2.0" edition = "2021" # Comet uses the same minimum Rust version as DataFusion rust-version = "1.88" [workspace.dependencies] arrow = { version = "58.1.0", features = ["prettyprint", "ffi", "chrono-tz"] } async-trait = { version = "0.1" } bytes = { version = "1.11.1" } parquet = { version = "58.1.0", default-features = false, features = ["experimental"] } datafusion = { version = "53.1.0", default-features = false, features = ["unicode_expressions", "crypto_expressions", "nested_expressions", "parquet"] } datafusion-datasource = { version = "53.1.0" } datafusion-physical-expr-adapter = { version = "53.1.0" } datafusion-spark = { version = "53.1.0", features = ["core"] } datafusion-comet-spark-expr = { path = "spark-expr" } datafusion-comet-common = { path = "common" } datafusion-comet-jni-bridge = { path = "jni-bridge" } datafusion-comet-proto = { path = "proto" } datafusion-comet-shuffle = { path = "shuffle" } chrono = { version = "0.4", default-features = false, features = ["clock"] } chrono-tz = { version = "0.10" } futures = "0.3.32" num = "0.4" rand = "0.10" regex = "1.12.3" thiserror = "2" object_store = { version = "0.13.1", features = ["gcp", "azure", "aws", "http"] } url = "2.2" aws-config = "1.8.14" aws-credential-types = "1.2.13" iceberg = { git = "https://github.com/apache/iceberg-rust", rev = "a2f067d" } iceberg-storage-opendal = { git = "https://github.com/apache/iceberg-rust", rev = "a2f067d", features = ["opendal-all"] } [profile.release] debug = true overflow-checks = false lto = "thin" codegen-units = 1 strip = "debuginfo" # CI profile: faster compilation, same overflow behavior as release # Use with: cargo build --profile ci [profile.ci] inherits = "release" lto = false # Skip LTO for faster linking codegen-units = 16 # Parallel codegen (faster compile, slightly larger binary) debug-assertions = true panic = "unwind" # Allow panics to be caught and logged across FFI boundary # overflow-checks inherited as false from release ================================================ FILE: native/README.md ================================================ # Apache DataFusion Comet Native Code This project contains the following crates: - [core](core): Native code used by the Comet Spark plugin - [proto](proto): Comet protocol buffer definition for query plans - [spark-expr](spark-expr): Spark-compatible DataFusion operators and expressions ================================================ FILE: native/common/Cargo.toml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. [package] name = "datafusion-comet-common" description = "Apache DataFusion Comet: common types shared across crates" version = { workspace = true } homepage = { workspace = true } repository = { workspace = true } authors = { workspace = true } readme = { workspace = true } license = { workspace = true } edition = { workspace = true } publish = false [dependencies] arrow = { workspace = true } datafusion = { workspace = true } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" thiserror = { workspace = true } [lib] name = "datafusion_comet_common" path = "src/lib.rs" [[bin]] name = "analyze_trace" path = "src/bin/analyze_trace.rs" ================================================ FILE: native/common/README.md ================================================ # datafusion-comet-common: Common Types This crate provides common types shared across Apache DataFusion Comet crates and is maintained as part of the [Apache DataFusion Comet] subproject. [Apache DataFusion Comet]: https://github.com/apache/datafusion-comet/ ================================================ FILE: native/common/src/bin/analyze_trace.rs ================================================ // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. //! Analyzes a Comet chrome trace event log (`comet-event-trace.json`) and //! compares jemalloc usage against the sum of per-thread Comet memory pool //! reservations. Reports any points where jemalloc exceeds the total pool size. //! //! Usage: //! cargo run --bin analyze_trace -- use serde::Deserialize; use std::collections::HashMap; use std::io::{BufRead, BufReader}; use std::{env, fs::File}; /// A single Chrome trace event (only the fields we care about). #[derive(Deserialize)] struct TraceEvent { name: String, ph: String, #[allow(dead_code)] tid: u64, ts: u64, #[serde(default)] args: HashMap, } /// Snapshot of memory state at a given timestamp. struct MemorySnapshot { ts: u64, jemalloc: u64, pool_total: u64, } fn format_bytes(bytes: u64) -> String { const MB: f64 = 1024.0 * 1024.0; format!("{:.1} MB", bytes as f64 / MB) } fn main() { let args: Vec = env::args().collect(); if args.len() != 2 { eprintln!("Usage: analyze_trace "); std::process::exit(1); } let file = File::open(&args[1]).expect("Failed to open trace file"); let reader = BufReader::new(file); // Latest jemalloc value (global, not per-thread) let mut latest_jemalloc: u64 = 0; // Per-thread pool reservations: thread_NNN -> bytes let mut pool_by_thread: HashMap = HashMap::new(); // Points where jemalloc exceeded pool total let mut violations: Vec = Vec::new(); // Track peak values let mut peak_jemalloc: u64 = 0; let mut peak_pool_total: u64 = 0; let mut peak_excess: u64 = 0; let mut counter_events: u64 = 0; // Each line is one JSON event, possibly with a trailing comma. // The file starts with "[ " on the first event line or as a prefix. for line in reader.lines() { let line = line.expect("Failed to read line"); let trimmed = line.trim(); // Skip empty lines or bare array brackets if trimmed.is_empty() || trimmed == "[" || trimmed == "]" { continue; } // Strip leading "[ " (first event) and trailing comma let json_str = trimmed .trim_start_matches("[ ") .trim_start_matches('[') .trim_end_matches(','); if json_str.is_empty() { continue; } // Only parse counter events (they contain "\"ph\": \"C\"") if !json_str.contains("\"ph\": \"C\"") { continue; } let event: TraceEvent = match serde_json::from_str(json_str) { Ok(e) => e, Err(_) => continue, }; if event.ph != "C" { continue; } counter_events += 1; if event.name == "jemalloc_allocated" { if let Some(val) = event.args.get("jemalloc_allocated") { latest_jemalloc = val.as_u64().unwrap_or(0); if latest_jemalloc > peak_jemalloc { peak_jemalloc = latest_jemalloc; } } } else if event.name.contains("comet_memory_reserved") { // Name format: thread_NNN_comet_memory_reserved let thread_key = event.name.clone(); if let Some(val) = event.args.get(&event.name) { let bytes = val.as_u64().unwrap_or(0); pool_by_thread.insert(thread_key, bytes); } } else { // Skip jvm_heap_used and other counters continue; } // After each jemalloc or pool update, check the current state let pool_total: u64 = pool_by_thread.values().sum(); if pool_total > peak_pool_total { peak_pool_total = pool_total; } if latest_jemalloc > 0 && pool_total > 0 && latest_jemalloc > pool_total { let excess = latest_jemalloc - pool_total; if excess > peak_excess { peak_excess = excess; } // Record violation (sample - don't record every single one) if violations.is_empty() || event.ts.saturating_sub(violations.last().unwrap().ts) > 1_000_000 || excess == peak_excess { violations.push(MemorySnapshot { ts: event.ts, jemalloc: latest_jemalloc, pool_total, }); } } } // Print summary println!("=== Comet Trace Memory Analysis ===\n"); println!("Counter events parsed: {counter_events}"); println!("Threads with memory pools: {}", pool_by_thread.len()); println!("Peak jemalloc allocated: {}", format_bytes(peak_jemalloc)); println!( "Peak pool total: {}", format_bytes(peak_pool_total) ); println!( "Peak excess (jemalloc - pool): {}", format_bytes(peak_excess) ); println!(); if violations.is_empty() { println!("OK: jemalloc never exceeded the total pool reservation."); } else { println!( "WARNING: jemalloc exceeded pool reservation at {} sampled points:\n", violations.len() ); println!( "{:>14} {:>14} {:>14} {:>14}", "Time (us)", "jemalloc", "pool_total", "excess" ); println!("{}", "-".repeat(62)); for snap in &violations { let excess = snap.jemalloc - snap.pool_total; println!( "{:>14} {:>14} {:>14} {:>14}", snap.ts, format_bytes(snap.jemalloc), format_bytes(snap.pool_total), format_bytes(excess), ); } } // Show final per-thread pool state println!("\n--- Final per-thread pool reservations ---\n"); let mut threads: Vec<_> = pool_by_thread.iter().collect(); threads.sort_by_key(|(k, _)| (*k).clone()); for (thread, bytes) in &threads { println!(" {thread}: {}", format_bytes(**bytes)); } println!("\n Total: {}", format_bytes(pool_by_thread.values().sum())); } ================================================ FILE: native/common/src/error.rs ================================================ // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. use arrow::error::ArrowError; use datafusion::common::DataFusionError; use std::sync::Arc; #[derive(thiserror::Error, Debug, Clone)] pub enum SparkError { // This list was generated from the Spark code. Many of the exceptions are not yet used by Comet #[error("[CAST_INVALID_INPUT] The value '{value}' of the type \"{from_type}\" cannot be cast to \"{to_type}\" \ because it is malformed. Correct the value as per the syntax, or change its target type. \ Use `try_cast` to tolerate malformed input and return NULL instead. If necessary \ set \"spark.sql.ansi.enabled\" to \"false\" to bypass this error.")] CastInvalidValue { value: String, from_type: String, to_type: String, }, /// Like CastInvalidValue but maps to SparkDateTimeException instead of SparkNumberFormatException. /// Used for string → timestamp/date cast failures. #[error("[CAST_INVALID_INPUT] The value '{value}' of the type \"{from_type}\" cannot be cast to \"{to_type}\" \ because it is malformed. Correct the value as per the syntax, or change its target type. \ Use `try_cast` to tolerate malformed input and return NULL instead. If necessary \ set \"spark.sql.ansi.enabled\" to \"false\" to bypass this error.")] InvalidInputInCastToDatetime { value: String, from_type: String, to_type: String, }, #[error("[NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION] {value} cannot be represented as Decimal({precision}, {scale}). If necessary set \"spark.sql.ansi.enabled\" to \"false\" to bypass this error, and return NULL instead.")] NumericValueOutOfRange { value: String, precision: u8, scale: i8, }, #[error("[NUMERIC_OUT_OF_SUPPORTED_RANGE] The value {value} cannot be interpreted as a numeric since it has more than 38 digits.")] NumericOutOfRange { value: String }, #[error("[CAST_OVERFLOW] The value {value} of the type \"{from_type}\" cannot be cast to \"{to_type}\" \ due to an overflow. Use `try_cast` to tolerate overflow and return NULL instead. If necessary \ set \"spark.sql.ansi.enabled\" to \"false\" to bypass this error.")] CastOverFlow { value: String, from_type: String, to_type: String, }, #[error("[CANNOT_PARSE_DECIMAL] Cannot parse decimal.")] CannotParseDecimal, #[error("[ARITHMETIC_OVERFLOW] {from_type} overflow. If necessary set \"spark.sql.ansi.enabled\" to \"false\" to bypass this error.")] ArithmeticOverflow { from_type: String }, #[error("[ARITHMETIC_OVERFLOW] Overflow in integral divide. Use `try_divide` to tolerate overflow and return NULL instead. If necessary set \"spark.sql.ansi.enabled\" to \"false\" to bypass this error.")] IntegralDivideOverflow, #[error("[ARITHMETIC_OVERFLOW] Overflow in sum of decimals. If necessary set \"spark.sql.ansi.enabled\" to \"false\" to bypass this error.")] DecimalSumOverflow, #[error("[DIVIDE_BY_ZERO] Division by zero. Use `try_divide` to tolerate divisor being 0 and return NULL instead. If necessary set \"spark.sql.ansi.enabled\" to \"false\" to bypass this error.")] DivideByZero, #[error("[REMAINDER_BY_ZERO] Division by zero. Use `try_remainder` to tolerate divisor being 0 and return NULL instead. If necessary set \"spark.sql.ansi.enabled\" to \"false\" to bypass this error.")] RemainderByZero, #[error("[INTERVAL_DIVIDED_BY_ZERO] Divide by zero in interval arithmetic.")] IntervalDividedByZero, #[error("[BINARY_ARITHMETIC_OVERFLOW] {value1} {symbol} {value2} caused overflow. Use `{function_name}` to tolerate overflow and return NULL instead. If necessary set \"spark.sql.ansi.enabled\" to \"false\" to bypass this error.")] BinaryArithmeticOverflow { value1: String, symbol: String, value2: String, function_name: String, }, #[error("[INTERVAL_ARITHMETIC_OVERFLOW.WITH_SUGGESTION] Interval arithmetic overflow. Use `{function_name}` to tolerate overflow and return NULL instead. If necessary set \"spark.sql.ansi.enabled\" to \"false\" to bypass this error.")] IntervalArithmeticOverflowWithSuggestion { function_name: String }, #[error("[INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION] Interval arithmetic overflow. If necessary set \"spark.sql.ansi.enabled\" to \"false\" to bypass this error.")] IntervalArithmeticOverflowWithoutSuggestion, #[error("[DATETIME_OVERFLOW] Datetime arithmetic overflow.")] DatetimeOverflow, #[error("[INVALID_ARRAY_INDEX] The index {index_value} is out of bounds. The array has {array_size} elements. Use the SQL function get() to tolerate accessing element at invalid index and return NULL instead. If necessary set \"spark.sql.ansi.enabled\" to \"false\" to bypass this error.")] InvalidArrayIndex { index_value: i32, array_size: i32 }, #[error("[INVALID_ARRAY_INDEX_IN_ELEMENT_AT] The index {index_value} is out of bounds. The array has {array_size} elements. Use try_element_at to tolerate accessing element at invalid index and return NULL instead. If necessary set \"spark.sql.ansi.enabled\" to \"false\" to bypass this error.")] InvalidElementAtIndex { index_value: i32, array_size: i32 }, #[error("[INVALID_BITMAP_POSITION] The bit position {bit_position} is out of bounds. The bitmap has {bitmap_num_bytes} bytes ({bitmap_num_bits} bits).")] InvalidBitmapPosition { bit_position: i64, bitmap_num_bytes: i64, bitmap_num_bits: i64, }, #[error("[INVALID_INDEX_OF_ZERO] The index 0 is invalid. An index shall be either < 0 or > 0 (the first element has index 1).")] InvalidIndexOfZero, #[error("[DUPLICATED_MAP_KEY] Cannot create map with duplicate keys: {key}.")] DuplicatedMapKey { key: String }, #[error("[NULL_MAP_KEY] Cannot use null as map key.")] NullMapKey, #[error("[MAP_KEY_VALUE_DIFF_SIZES] The key array and value array of a map must have the same length.")] MapKeyValueDiffSizes, #[error("[EXCEED_LIMIT_LENGTH] Cannot create a map with {size} elements which exceeds the limit {max_size}.")] ExceedMapSizeLimit { size: i32, max_size: i32 }, #[error("[COLLECTION_SIZE_LIMIT_EXCEEDED] Cannot create array with {num_elements} elements which exceeds the limit {max_elements}.")] CollectionSizeLimitExceeded { num_elements: i64, max_elements: i64, }, #[error("[NOT_NULL_ASSERT_VIOLATION] The field `{field_name}` cannot be null.")] NotNullAssertViolation { field_name: String }, #[error("[VALUE_IS_NULL] The value of field `{field_name}` at row {row_index} is null.")] ValueIsNull { field_name: String, row_index: i32 }, #[error("[CANNOT_PARSE_TIMESTAMP] Cannot parse timestamp: {message}. Try using `{suggested_func}` instead.")] CannotParseTimestamp { message: String, suggested_func: String, }, #[error("[INVALID_FRACTION_OF_SECOND] The fraction of second {value} is invalid. Valid values are in the range [0, 60]. If necessary set \"spark.sql.ansi.enabled\" to \"false\" to bypass this error.")] InvalidFractionOfSecond { value: f64 }, #[error("[INVALID_UTF8_STRING] Invalid UTF-8 string: {hex_string}.")] InvalidUtf8String { hex_string: String }, #[error("[UNEXPECTED_POSITIVE_VALUE] The {parameter_name} parameter must be less than or equal to 0. The actual value is {actual_value}.")] UnexpectedPositiveValue { parameter_name: String, actual_value: i32, }, #[error("[UNEXPECTED_NEGATIVE_VALUE] The {parameter_name} parameter must be greater than or equal to 0. The actual value is {actual_value}.")] UnexpectedNegativeValue { parameter_name: String, actual_value: i32, }, #[error("[INVALID_PARAMETER_VALUE] Invalid regex group index {group_index} in function `{function_name}`. Group count is {group_count}.")] InvalidRegexGroupIndex { function_name: String, group_count: i32, group_index: i32, }, #[error("[DATATYPE_CANNOT_ORDER] Cannot order by type: {data_type}.")] DatatypeCannotOrder { data_type: String }, #[error("[SCALAR_SUBQUERY_TOO_MANY_ROWS] Scalar subquery returned more than one row.")] ScalarSubqueryTooManyRows, #[error("{message}")] FileNotFound { message: String }, #[error("[_LEGACY_ERROR_TEMP_2093] Found duplicate field(s) \"{required_field_name}\": [{matched_fields}] in case-insensitive mode")] DuplicateFieldCaseInsensitive { required_field_name: String, matched_fields: String, }, #[error("ArrowError: {0}.")] Arrow(Arc), #[error("InternalError: {0}.")] Internal(String), } impl SparkError { /// Serialize this error to JSON format for JNI transfer pub fn to_json(&self) -> String { let error_class = self.error_class().unwrap_or(""); // Create a JSON structure with errorType, errorClass, and params match serde_json::to_string(&serde_json::json!({ "errorType": self.error_type_name(), "errorClass": error_class, "params": self.params_as_json(), })) { Ok(json) => json, Err(e) => { // Fallback if serialization fails format!( "{{\"errorType\":\"SerializationError\",\"message\":\"{}\"}}", e ) } } } /// Get the error type name for JSON serialization pub(crate) fn error_type_name(&self) -> &'static str { match self { SparkError::CastInvalidValue { .. } => "CastInvalidValue", SparkError::InvalidInputInCastToDatetime { .. } => "InvalidInputInCastToDatetime", SparkError::NumericValueOutOfRange { .. } => "NumericValueOutOfRange", SparkError::NumericOutOfRange { .. } => "NumericOutOfRange", SparkError::CastOverFlow { .. } => "CastOverFlow", SparkError::CannotParseDecimal => "CannotParseDecimal", SparkError::ArithmeticOverflow { .. } => "ArithmeticOverflow", SparkError::IntegralDivideOverflow => "IntegralDivideOverflow", SparkError::DecimalSumOverflow => "DecimalSumOverflow", SparkError::DivideByZero => "DivideByZero", SparkError::RemainderByZero => "RemainderByZero", SparkError::IntervalDividedByZero => "IntervalDividedByZero", SparkError::BinaryArithmeticOverflow { .. } => "BinaryArithmeticOverflow", SparkError::IntervalArithmeticOverflowWithSuggestion { .. } => { "IntervalArithmeticOverflowWithSuggestion" } SparkError::IntervalArithmeticOverflowWithoutSuggestion => { "IntervalArithmeticOverflowWithoutSuggestion" } SparkError::DatetimeOverflow => "DatetimeOverflow", SparkError::InvalidArrayIndex { .. } => "InvalidArrayIndex", SparkError::InvalidElementAtIndex { .. } => "InvalidElementAtIndex", SparkError::InvalidBitmapPosition { .. } => "InvalidBitmapPosition", SparkError::InvalidIndexOfZero => "InvalidIndexOfZero", SparkError::DuplicatedMapKey { .. } => "DuplicatedMapKey", SparkError::NullMapKey => "NullMapKey", SparkError::MapKeyValueDiffSizes => "MapKeyValueDiffSizes", SparkError::ExceedMapSizeLimit { .. } => "ExceedMapSizeLimit", SparkError::CollectionSizeLimitExceeded { .. } => "CollectionSizeLimitExceeded", SparkError::NotNullAssertViolation { .. } => "NotNullAssertViolation", SparkError::ValueIsNull { .. } => "ValueIsNull", SparkError::CannotParseTimestamp { .. } => "CannotParseTimestamp", SparkError::InvalidFractionOfSecond { .. } => "InvalidFractionOfSecond", SparkError::InvalidUtf8String { .. } => "InvalidUtf8String", SparkError::UnexpectedPositiveValue { .. } => "UnexpectedPositiveValue", SparkError::UnexpectedNegativeValue { .. } => "UnexpectedNegativeValue", SparkError::InvalidRegexGroupIndex { .. } => "InvalidRegexGroupIndex", SparkError::DatatypeCannotOrder { .. } => "DatatypeCannotOrder", SparkError::ScalarSubqueryTooManyRows => "ScalarSubqueryTooManyRows", SparkError::FileNotFound { .. } => "FileNotFound", SparkError::DuplicateFieldCaseInsensitive { .. } => "DuplicateFieldCaseInsensitive", SparkError::Arrow(_) => "Arrow", SparkError::Internal(_) => "Internal", } } /// Extract parameters as JSON value pub(crate) fn params_as_json(&self) -> serde_json::Value { match self { SparkError::CastInvalidValue { value, from_type, to_type, } => { serde_json::json!({ "value": value, "fromType": from_type, "toType": to_type, }) } SparkError::InvalidInputInCastToDatetime { value, from_type, to_type, } => { serde_json::json!({ "value": value, "fromType": from_type, "toType": to_type, }) } SparkError::NumericValueOutOfRange { value, precision, scale, } => { serde_json::json!({ "value": value, "precision": precision, "scale": scale, }) } SparkError::NumericOutOfRange { value } => { serde_json::json!({ "value": value, }) } SparkError::CastOverFlow { value, from_type, to_type, } => { serde_json::json!({ "value": value, "fromType": from_type, "toType": to_type, }) } SparkError::ArithmeticOverflow { from_type } => { serde_json::json!({ "fromType": from_type, }) } SparkError::BinaryArithmeticOverflow { value1, symbol, value2, function_name, } => { serde_json::json!({ "value1": value1, "symbol": symbol, "value2": value2, "functionName": function_name, }) } SparkError::IntervalArithmeticOverflowWithSuggestion { function_name } => { serde_json::json!({ "functionName": function_name, }) } SparkError::InvalidArrayIndex { index_value, array_size, } => { serde_json::json!({ "indexValue": index_value, "arraySize": array_size, }) } SparkError::InvalidElementAtIndex { index_value, array_size, } => { serde_json::json!({ "indexValue": index_value, "arraySize": array_size, }) } SparkError::InvalidBitmapPosition { bit_position, bitmap_num_bytes, bitmap_num_bits, } => { serde_json::json!({ "bitPosition": bit_position, "bitmapNumBytes": bitmap_num_bytes, "bitmapNumBits": bitmap_num_bits, }) } SparkError::DuplicatedMapKey { key } => { serde_json::json!({ "key": key, }) } SparkError::ExceedMapSizeLimit { size, max_size } => { serde_json::json!({ "size": size, "maxSize": max_size, }) } SparkError::CollectionSizeLimitExceeded { num_elements, max_elements, } => { serde_json::json!({ "numElements": num_elements, "maxElements": max_elements, }) } SparkError::NotNullAssertViolation { field_name } => { serde_json::json!({ "fieldName": field_name, }) } SparkError::ValueIsNull { field_name, row_index, } => { serde_json::json!({ "fieldName": field_name, "rowIndex": row_index, }) } SparkError::CannotParseTimestamp { message, suggested_func, } => { serde_json::json!({ "message": message, "suggestedFunc": suggested_func, }) } SparkError::InvalidFractionOfSecond { value } => { serde_json::json!({ "value": value, }) } SparkError::InvalidUtf8String { hex_string } => { serde_json::json!({ "hexString": hex_string, }) } SparkError::UnexpectedPositiveValue { parameter_name, actual_value, } => { serde_json::json!({ "parameterName": parameter_name, "actualValue": actual_value, }) } SparkError::UnexpectedNegativeValue { parameter_name, actual_value, } => { serde_json::json!({ "parameterName": parameter_name, "actualValue": actual_value, }) } SparkError::InvalidRegexGroupIndex { function_name, group_count, group_index, } => { serde_json::json!({ "functionName": function_name, "groupCount": group_count, "groupIndex": group_index, }) } SparkError::DatatypeCannotOrder { data_type } => { serde_json::json!({ "dataType": data_type, }) } SparkError::FileNotFound { message } => { serde_json::json!({ "message": message, }) } SparkError::DuplicateFieldCaseInsensitive { required_field_name, matched_fields, } => { serde_json::json!({ "requiredFieldName": required_field_name, "matchedOrcFields": matched_fields, }) } SparkError::Arrow(e) => { serde_json::json!({ "message": e.to_string(), }) } SparkError::Internal(msg) => { serde_json::json!({ "message": msg, }) } // Simple errors with no parameters _ => serde_json::json!({}), } } /// Returns the appropriate Spark exception class for this error pub fn exception_class(&self) -> &'static str { match self { // ArithmeticException SparkError::DivideByZero | SparkError::RemainderByZero | SparkError::IntervalDividedByZero | SparkError::NumericValueOutOfRange { .. } | SparkError::NumericOutOfRange { .. } // Comet-specific extension | SparkError::ArithmeticOverflow { .. } | SparkError::IntegralDivideOverflow | SparkError::DecimalSumOverflow | SparkError::BinaryArithmeticOverflow { .. } | SparkError::IntervalArithmeticOverflowWithSuggestion { .. } | SparkError::IntervalArithmeticOverflowWithoutSuggestion | SparkError::DatetimeOverflow => "org/apache/spark/SparkArithmeticException", // CastOverflow gets special handling with CastOverflowException SparkError::CastOverFlow { .. } => "org/apache/spark/sql/comet/CastOverflowException", // NumberFormatException (for cast invalid input errors) SparkError::CastInvalidValue { .. } => "org/apache/spark/SparkNumberFormatException", // ArrayIndexOutOfBoundsException SparkError::InvalidArrayIndex { .. } | SparkError::InvalidElementAtIndex { .. } | SparkError::InvalidBitmapPosition { .. } | SparkError::InvalidIndexOfZero => "org/apache/spark/SparkArrayIndexOutOfBoundsException", // RuntimeException SparkError::CannotParseDecimal | SparkError::DuplicatedMapKey { .. } | SparkError::NullMapKey | SparkError::MapKeyValueDiffSizes | SparkError::ExceedMapSizeLimit { .. } | SparkError::CollectionSizeLimitExceeded { .. } | SparkError::NotNullAssertViolation { .. } | SparkError::ValueIsNull { .. } // Comet-specific extension | SparkError::UnexpectedPositiveValue { .. } | SparkError::UnexpectedNegativeValue { .. } | SparkError::InvalidRegexGroupIndex { .. } | SparkError::ScalarSubqueryTooManyRows => "org/apache/spark/SparkRuntimeException", // DateTimeException SparkError::InvalidInputInCastToDatetime { .. } | SparkError::CannotParseTimestamp { .. } | SparkError::InvalidFractionOfSecond { .. } => "org/apache/spark/SparkDateTimeException", // IllegalArgumentException SparkError::DatatypeCannotOrder { .. } | SparkError::InvalidUtf8String { .. } => "org/apache/spark/SparkIllegalArgumentException", // FileNotFound - will be converted to SparkFileNotFoundException by the shim SparkError::FileNotFound { .. } => "org/apache/spark/SparkException", // DuplicateFieldCaseInsensitive - converted to SparkRuntimeException by the shim SparkError::DuplicateFieldCaseInsensitive { .. } => { "org/apache/spark/SparkRuntimeException" } // Generic errors SparkError::Arrow(_) | SparkError::Internal(_) => "org/apache/spark/SparkException", } } /// Returns the Spark error class code for this error pub(crate) fn error_class(&self) -> Option<&'static str> { match self { // Cast errors SparkError::CastInvalidValue { .. } => Some("CAST_INVALID_INPUT"), SparkError::InvalidInputInCastToDatetime { .. } => Some("CAST_INVALID_INPUT"), SparkError::CastOverFlow { .. } => Some("CAST_OVERFLOW"), SparkError::NumericValueOutOfRange { .. } => { Some("NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION") } SparkError::NumericOutOfRange { .. } => Some("NUMERIC_OUT_OF_SUPPORTED_RANGE"), SparkError::CannotParseDecimal => Some("CANNOT_PARSE_DECIMAL"), // Arithmetic errors SparkError::DivideByZero => Some("DIVIDE_BY_ZERO"), SparkError::RemainderByZero => Some("REMAINDER_BY_ZERO"), SparkError::IntervalDividedByZero => Some("INTERVAL_DIVIDED_BY_ZERO"), SparkError::ArithmeticOverflow { .. } => Some("ARITHMETIC_OVERFLOW"), SparkError::IntegralDivideOverflow => Some("ARITHMETIC_OVERFLOW"), SparkError::DecimalSumOverflow => Some("ARITHMETIC_OVERFLOW"), SparkError::BinaryArithmeticOverflow { .. } => Some("BINARY_ARITHMETIC_OVERFLOW"), SparkError::IntervalArithmeticOverflowWithSuggestion { .. } => { Some("INTERVAL_ARITHMETIC_OVERFLOW") } SparkError::IntervalArithmeticOverflowWithoutSuggestion => { Some("INTERVAL_ARITHMETIC_OVERFLOW") } SparkError::DatetimeOverflow => Some("DATETIME_OVERFLOW"), // Array index errors SparkError::InvalidArrayIndex { .. } => Some("INVALID_ARRAY_INDEX"), SparkError::InvalidElementAtIndex { .. } => Some("INVALID_ARRAY_INDEX_IN_ELEMENT_AT"), SparkError::InvalidBitmapPosition { .. } => Some("INVALID_BITMAP_POSITION"), SparkError::InvalidIndexOfZero => Some("INVALID_INDEX_OF_ZERO"), // Map/Collection errors SparkError::DuplicatedMapKey { .. } => Some("DUPLICATED_MAP_KEY"), SparkError::NullMapKey => Some("NULL_MAP_KEY"), SparkError::MapKeyValueDiffSizes => Some("MAP_KEY_VALUE_DIFF_SIZES"), SparkError::ExceedMapSizeLimit { .. } => Some("EXCEED_LIMIT_LENGTH"), SparkError::CollectionSizeLimitExceeded { .. } => { Some("COLLECTION_SIZE_LIMIT_EXCEEDED") } // Null validation errors SparkError::NotNullAssertViolation { .. } => Some("NOT_NULL_ASSERT_VIOLATION"), SparkError::ValueIsNull { .. } => Some("VALUE_IS_NULL"), // DateTime errors SparkError::CannotParseTimestamp { .. } => Some("CANNOT_PARSE_TIMESTAMP"), SparkError::InvalidFractionOfSecond { .. } => Some("INVALID_FRACTION_OF_SECOND"), // String/UTF8 errors SparkError::InvalidUtf8String { .. } => Some("INVALID_UTF8_STRING"), // Function parameter errors SparkError::UnexpectedPositiveValue { .. } => Some("UNEXPECTED_POSITIVE_VALUE"), SparkError::UnexpectedNegativeValue { .. } => Some("UNEXPECTED_NEGATIVE_VALUE"), // Regex errors SparkError::InvalidRegexGroupIndex { .. } => Some("INVALID_PARAMETER_VALUE"), // Unsupported operation errors SparkError::DatatypeCannotOrder { .. } => Some("DATATYPE_CANNOT_ORDER"), // Subquery errors SparkError::ScalarSubqueryTooManyRows => Some("SCALAR_SUBQUERY_TOO_MANY_ROWS"), // File not found SparkError::FileNotFound { .. } => Some("_LEGACY_ERROR_TEMP_2055"), // Duplicate field in case-insensitive mode SparkError::DuplicateFieldCaseInsensitive { .. } => Some("_LEGACY_ERROR_TEMP_2093"), // Generic errors (no error class) SparkError::Arrow(_) | SparkError::Internal(_) => None, } } } pub type SparkResult = Result; /// Convert decimal overflow to SparkError::NumericValueOutOfRange. pub fn decimal_overflow_error(value: i128, precision: u8, scale: i8) -> SparkError { SparkError::NumericValueOutOfRange { value: value.to_string(), precision, scale, } } /// Wrapper that adds QueryContext to SparkError /// /// This allows attaching SQL context information (query text, line/position, object name) to errors #[derive(Debug, Clone)] pub struct SparkErrorWithContext { /// The underlying SparkError pub error: SparkError, /// Optional QueryContext for SQL location information pub context: Option>, } impl SparkErrorWithContext { /// Create a SparkErrorWithContext without context pub fn new(error: SparkError) -> Self { Self { error, context: None, } } /// Create a SparkErrorWithContext with QueryContext pub fn with_context(error: SparkError, context: Arc) -> Self { Self { error, context: Some(context), } } /// Serialize to JSON including optional context field pub fn to_json(&self) -> String { let mut json_obj = serde_json::json!({ "errorType": self.error.error_type_name(), "errorClass": self.error.error_class().unwrap_or(""), "params": self.error.params_as_json(), }); if let Some(ctx) = &self.context { // Serialize context fields json_obj["context"] = serde_json::json!({ "sqlText": ctx.sql_text.as_str(), "startIndex": ctx.start_index, "stopIndex": ctx.stop_index, "objectType": ctx.object_type, "objectName": ctx.object_name, "line": ctx.line, "startPosition": ctx.start_position, }); // Add formatted summary json_obj["summary"] = serde_json::json!(ctx.format_summary()); } serde_json::to_string(&json_obj).unwrap_or_else(|e| { format!( "{{\"errorType\":\"SerializationError\",\"message\":\"{}\"}}", e ) }) } } impl std::fmt::Display for SparkErrorWithContext { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.error)?; if let Some(ctx) = &self.context { write!(f, "\n{}", ctx.format_summary())?; } Ok(()) } } impl std::error::Error for SparkErrorWithContext {} impl From for SparkErrorWithContext { fn from(error: SparkError) -> Self { SparkErrorWithContext::new(error) } } impl From for DataFusionError { fn from(value: SparkErrorWithContext) -> Self { DataFusionError::External(Box::new(value)) } } impl From for SparkError { fn from(value: ArrowError) -> Self { SparkError::Arrow(Arc::new(value)) } } impl From for DataFusionError { fn from(value: SparkError) -> Self { DataFusionError::External(Box::new(value)) } } #[cfg(test)] mod tests { use super::*; #[test] fn test_divide_by_zero_json() { let error = SparkError::DivideByZero; let json = error.to_json(); assert!(json.contains("\"errorType\":\"DivideByZero\"")); assert!(json.contains("\"errorClass\":\"DIVIDE_BY_ZERO\"")); // Verify it's valid JSON let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); assert_eq!(parsed["errorType"], "DivideByZero"); assert_eq!(parsed["errorClass"], "DIVIDE_BY_ZERO"); } #[test] fn test_remainder_by_zero_json() { let error = SparkError::RemainderByZero; let json = error.to_json(); assert!(json.contains("\"errorType\":\"RemainderByZero\"")); assert!(json.contains("\"errorClass\":\"REMAINDER_BY_ZERO\"")); } #[test] fn test_binary_overflow_json() { let error = SparkError::BinaryArithmeticOverflow { value1: "32767".to_string(), symbol: "+".to_string(), value2: "1".to_string(), function_name: "try_add".to_string(), }; let json = error.to_json(); // Verify it's valid JSON let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); assert_eq!(parsed["errorType"], "BinaryArithmeticOverflow"); assert_eq!(parsed["errorClass"], "BINARY_ARITHMETIC_OVERFLOW"); assert_eq!(parsed["params"]["value1"], "32767"); assert_eq!(parsed["params"]["symbol"], "+"); assert_eq!(parsed["params"]["value2"], "1"); assert_eq!(parsed["params"]["functionName"], "try_add"); } #[test] fn test_invalid_array_index_json() { let error = SparkError::InvalidArrayIndex { index_value: 10, array_size: 3, }; let json = error.to_json(); let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); assert_eq!(parsed["errorType"], "InvalidArrayIndex"); assert_eq!(parsed["errorClass"], "INVALID_ARRAY_INDEX"); assert_eq!(parsed["params"]["indexValue"], 10); assert_eq!(parsed["params"]["arraySize"], 3); } #[test] fn test_numeric_value_out_of_range_json() { let error = SparkError::NumericValueOutOfRange { value: "999.99".to_string(), precision: 5, scale: 2, }; let json = error.to_json(); let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); assert_eq!(parsed["errorType"], "NumericValueOutOfRange"); assert_eq!( parsed["errorClass"], "NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION" ); assert_eq!(parsed["params"]["value"], "999.99"); assert_eq!(parsed["params"]["precision"], 5); assert_eq!(parsed["params"]["scale"], 2); } #[test] fn test_cast_invalid_value_json() { let error = SparkError::CastInvalidValue { value: "abc".to_string(), from_type: "STRING".to_string(), to_type: "INT".to_string(), }; let json = error.to_json(); let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); assert_eq!(parsed["errorType"], "CastInvalidValue"); assert_eq!(parsed["errorClass"], "CAST_INVALID_INPUT"); assert_eq!(parsed["params"]["value"], "abc"); assert_eq!(parsed["params"]["fromType"], "STRING"); assert_eq!(parsed["params"]["toType"], "INT"); } #[test] fn test_duplicated_map_key_json() { let error = SparkError::DuplicatedMapKey { key: "duplicate_key".to_string(), }; let json = error.to_json(); let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); assert_eq!(parsed["errorType"], "DuplicatedMapKey"); assert_eq!(parsed["errorClass"], "DUPLICATED_MAP_KEY"); assert_eq!(parsed["params"]["key"], "duplicate_key"); } #[test] fn test_null_map_key_json() { let error = SparkError::NullMapKey; let json = error.to_json(); let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); assert_eq!(parsed["errorType"], "NullMapKey"); assert_eq!(parsed["errorClass"], "NULL_MAP_KEY"); // Params should be an empty object assert_eq!(parsed["params"], serde_json::json!({})); } #[test] fn test_error_class_mapping() { // Test that error_class() returns the correct error class assert_eq!( SparkError::DivideByZero.error_class(), Some("DIVIDE_BY_ZERO") ); assert_eq!( SparkError::RemainderByZero.error_class(), Some("REMAINDER_BY_ZERO") ); assert_eq!( SparkError::InvalidArrayIndex { index_value: 0, array_size: 0 } .error_class(), Some("INVALID_ARRAY_INDEX") ); assert_eq!(SparkError::NullMapKey.error_class(), Some("NULL_MAP_KEY")); } #[test] fn test_exception_class_mapping() { // Test that exception_class() returns the correct Java exception class assert_eq!( SparkError::DivideByZero.exception_class(), "org/apache/spark/SparkArithmeticException" ); assert_eq!( SparkError::InvalidArrayIndex { index_value: 0, array_size: 0 } .exception_class(), "org/apache/spark/SparkArrayIndexOutOfBoundsException" ); assert_eq!( SparkError::NullMapKey.exception_class(), "org/apache/spark/SparkRuntimeException" ); } } ================================================ FILE: native/common/src/lib.rs ================================================ // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. mod error; mod query_context; pub mod tracing; mod utils; pub use error::{decimal_overflow_error, SparkError, SparkErrorWithContext, SparkResult}; pub use query_context::{create_query_context_map, QueryContext, QueryContextMap}; pub use utils::bytes_to_i128; ================================================ FILE: native/common/src/query_context.rs ================================================ // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. //! Query execution context for error reporting //! //! This module provides QueryContext which mirrors Spark's SQLQueryContext //! for providing SQL text, line/position information, and error location //! pointers in exception messages. use serde::{Deserialize, Serialize}; use std::sync::Arc; /// Based on Spark's SQLQueryContext for error reporting. /// /// Contains information about where an error occurred in a SQL query, /// including the full SQL text, line/column positions, and object context. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct QueryContext { /// Full SQL query text #[serde(rename = "sqlText")] pub sql_text: Arc, /// Start offset in SQL text (0-based, character index) #[serde(rename = "startIndex")] pub start_index: i32, /// Stop offset in SQL text (0-based, character index, inclusive) #[serde(rename = "stopIndex")] pub stop_index: i32, /// Object type (e.g., "VIEW", "Project", "Filter") #[serde(rename = "objectType", skip_serializing_if = "Option::is_none")] pub object_type: Option, /// Object name (e.g., view name, column name) #[serde(rename = "objectName", skip_serializing_if = "Option::is_none")] pub object_name: Option, /// Line number in SQL query (1-based) pub line: i32, /// Column position within the line (0-based) #[serde(rename = "startPosition")] pub start_position: i32, } impl QueryContext { #[allow(clippy::too_many_arguments)] pub fn new( sql_text: String, start_index: i32, stop_index: i32, object_type: Option, object_name: Option, line: i32, start_position: i32, ) -> Self { Self { sql_text: Arc::new(sql_text), start_index, stop_index, object_type, object_name, line, start_position, } } /// Convert a character index to a byte offset in the SQL text. /// Returns None if the character index is out of range. fn char_index_to_byte_offset(&self, char_index: usize) -> Option { self.sql_text .char_indices() .nth(char_index) .map(|(byte_offset, _)| byte_offset) } /// Generate a summary string showing SQL fragment with error location. /// (From SQLQueryContext.summary) /// /// Format example: /// ```text /// == SQL of VIEW v1 (line 1, position 8) == /// SELECT a/b FROM t /// ^^^ /// ``` pub fn format_summary(&self) -> String { let start_char = self.start_index.max(0) as usize; // stop_index is inclusive; fragment covers [start, stop] let stop_char = (self.stop_index + 1).max(0) as usize; let fragment = match ( self.char_index_to_byte_offset(start_char), // stop_char may equal sql_text.chars().count() (one past the end) self.char_index_to_byte_offset(stop_char).or_else(|| { if stop_char == self.sql_text.chars().count() { Some(self.sql_text.len()) } else { None } }), ) { (Some(start_byte), Some(stop_byte)) => &self.sql_text[start_byte..stop_byte], _ => "", }; // Build the header line let mut summary = String::from("== SQL"); if let Some(obj_type) = &self.object_type { if !obj_type.is_empty() { summary.push_str(" of "); summary.push_str(obj_type); if let Some(obj_name) = &self.object_name { if !obj_name.is_empty() { summary.push(' '); summary.push_str(obj_name); } } } } summary.push_str(&format!( " (line {}, position {}) ==\n", self.line, self.start_position + 1 // Convert 0-based to 1-based for display )); // Add the SQL text with fragment highlighted summary.push_str(&self.sql_text); summary.push('\n'); // Add caret pointer let caret_position = self.start_position.max(0) as usize; summary.push_str(&" ".repeat(caret_position)); // fragment.chars().count() gives the correct display width for non-ASCII summary.push_str(&"^".repeat(fragment.chars().count().max(1))); summary } /// Returns the SQL fragment that caused the error. #[cfg(test)] fn fragment(&self) -> String { let start_char = self.start_index.max(0) as usize; let stop_char = (self.stop_index + 1).max(0) as usize; match ( self.char_index_to_byte_offset(start_char), self.char_index_to_byte_offset(stop_char).or_else(|| { if stop_char == self.sql_text.chars().count() { Some(self.sql_text.len()) } else { None } }), ) { (Some(start_byte), Some(stop_byte)) => self.sql_text[start_byte..stop_byte].to_string(), _ => String::new(), } } } use std::collections::HashMap; use std::sync::RwLock; /// Map that stores QueryContext information for expressions during execution. /// /// This map is populated during plan deserialization and accessed /// during error creation to attach SQL context to exceptions. #[derive(Debug)] pub struct QueryContextMap { /// Map from expression ID to QueryContext contexts: RwLock>>, } impl QueryContextMap { pub fn new() -> Self { Self { contexts: RwLock::new(HashMap::new()), } } /// Register a QueryContext for an expression ID. /// /// If the expression ID already exists, it will be replaced. /// /// # Arguments /// * `expr_id` - Unique expression identifier from protobuf /// * `context` - QueryContext containing SQL text and position info pub fn register(&self, expr_id: u64, context: QueryContext) { let mut contexts = self.contexts.write().unwrap(); contexts.insert(expr_id, Arc::new(context)); } /// Get the QueryContext for an expression ID. /// /// Returns None if no context is registered for this expression. /// /// # Arguments /// * `expr_id` - Expression identifier to look up pub fn get(&self, expr_id: u64) -> Option> { let contexts = self.contexts.read().unwrap(); contexts.get(&expr_id).cloned() } /// Clear all registered contexts. /// /// This is typically called after plan execution completes to free memory. pub fn clear(&self) { let mut contexts = self.contexts.write().unwrap(); contexts.clear(); } /// Return the number of registered contexts (for debugging/testing) pub fn len(&self) -> usize { let contexts = self.contexts.read().unwrap(); contexts.len() } /// Check if the map is empty pub fn is_empty(&self) -> bool { self.len() == 0 } } impl Default for QueryContextMap { fn default() -> Self { Self::new() } } /// Create a new session-scoped QueryContextMap. /// /// This should be called once per SessionContext during plan creation /// and passed to expressions that need query context for error reporting. pub fn create_query_context_map() -> Arc { Arc::new(QueryContextMap::new()) } #[cfg(test)] mod tests { use super::*; #[test] fn test_query_context_creation() { let ctx = QueryContext::new( "SELECT a/b FROM t".to_string(), 7, 9, Some("Divide".to_string()), Some("a/b".to_string()), 1, 7, ); assert_eq!(*ctx.sql_text, "SELECT a/b FROM t"); assert_eq!(ctx.start_index, 7); assert_eq!(ctx.stop_index, 9); assert_eq!(ctx.object_type, Some("Divide".to_string())); assert_eq!(ctx.object_name, Some("a/b".to_string())); assert_eq!(ctx.line, 1); assert_eq!(ctx.start_position, 7); } #[test] fn test_query_context_serialization() { let ctx = QueryContext::new( "SELECT a/b FROM t".to_string(), 7, 9, Some("Divide".to_string()), Some("a/b".to_string()), 1, 7, ); let json = serde_json::to_string(&ctx).unwrap(); let deserialized: QueryContext = serde_json::from_str(&json).unwrap(); assert_eq!(ctx, deserialized); } #[test] fn test_format_summary() { let ctx = QueryContext::new( "SELECT a/b FROM t".to_string(), 7, 9, Some("VIEW".to_string()), Some("v1".to_string()), 1, 7, ); let summary = ctx.format_summary(); assert!(summary.contains("== SQL of VIEW v1 (line 1, position 8) ==")); assert!(summary.contains("SELECT a/b FROM t")); assert!(summary.contains("^^^")); // Three carets for "a/b" } #[test] fn test_format_summary_without_object() { let ctx = QueryContext::new("SELECT a/b FROM t".to_string(), 7, 9, None, None, 1, 7); let summary = ctx.format_summary(); assert!(summary.contains("== SQL (line 1, position 8) ==")); assert!(summary.contains("SELECT a/b FROM t")); } #[test] fn test_fragment() { let ctx = QueryContext::new("SELECT a/b FROM t".to_string(), 7, 9, None, None, 1, 7); assert_eq!(ctx.fragment(), "a/b"); } #[test] fn test_arc_string_sharing() { let ctx1 = QueryContext::new("SELECT a/b FROM t".to_string(), 7, 9, None, None, 1, 7); let ctx2 = ctx1.clone(); // Arc should share the same allocation assert!(Arc::ptr_eq(&ctx1.sql_text, &ctx2.sql_text)); } #[test] fn test_json_with_optional_fields() { let ctx = QueryContext::new("SELECT a/b FROM t".to_string(), 7, 9, None, None, 1, 7); let json = serde_json::to_string(&ctx).unwrap(); // Should not serialize objectType and objectName when None assert!(!json.contains("objectType")); assert!(!json.contains("objectName")); } #[test] fn test_map_register_and_get() { let map = QueryContextMap::new(); let ctx = QueryContext::new("SELECT a/b FROM t".to_string(), 7, 9, None, None, 1, 7); map.register(1, ctx.clone()); let retrieved = map.get(1).unwrap(); assert_eq!(*retrieved.sql_text, "SELECT a/b FROM t"); assert_eq!(retrieved.start_index, 7); } #[test] fn test_map_get_nonexistent() { let map = QueryContextMap::new(); assert!(map.get(999).is_none()); } #[test] fn test_map_clear() { let map = QueryContextMap::new(); let ctx = QueryContext::new("SELECT a/b FROM t".to_string(), 7, 9, None, None, 1, 7); map.register(1, ctx); assert_eq!(map.len(), 1); map.clear(); assert_eq!(map.len(), 0); assert!(map.is_empty()); } // Verify that fragment() and format_summary() correctly handle SQL text that // contains multi-byte characters #[test] fn test_fragment_non_ascii_accented() { // "é" is a 2-byte UTF-8 sequence (U+00E9). // SQL: "SELECT café FROM t" // 0123456789... // char indices: c=7, a=8, f=9, é=10, ' '=11 ... FROM = 12.. // start_index=7, stop_index=10 should yield "café" let sql = "SELECT café FROM t".to_string(); let ctx = QueryContext::new(sql, 7, 10, None, None, 1, 7); assert_eq!(ctx.fragment(), "café"); } } ================================================ FILE: native/common/src/tracing.rs ================================================ // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. use datafusion::common::instant::Instant; use std::fs::{File, OpenOptions}; use std::io::{BufWriter, Write}; use std::sync::{Arc, LazyLock, Mutex}; pub static RECORDER: LazyLock = LazyLock::new(Recorder::new); /// Log events using Chrome trace format JSON /// https://github.com/catapult-project/catapult/blob/main/tracing/README.md pub struct Recorder { now: Instant, writer: Arc>>, } impl Default for Recorder { fn default() -> Self { Self::new() } } impl Recorder { pub fn new() -> Self { let file = OpenOptions::new() .create(true) .append(true) .open("comet-event-trace.json") .expect("Error writing tracing"); let mut writer = BufWriter::new(file); // Write start of JSON array. Note that there is no requirement to write // the closing ']'. writer .write_all("[ ".as_bytes()) .expect("Error writing tracing"); Self { now: Instant::now(), writer: Arc::new(Mutex::new(writer)), } } pub fn begin_task(&self, name: &str) { self.log_event(name, "B") } pub fn end_task(&self, name: &str) { self.log_event(name, "E") } pub fn log_memory_usage(&self, name: &str, usage_bytes: u64) { let json = format!( "{{ \"name\": \"{name}\", \"cat\": \"PERF\", \"ph\": \"C\", \"pid\": 1, \"tid\": {}, \"ts\": {}, \"args\": {{ \"{name}\": {usage_bytes} }} }},\n", get_thread_id(), self.now.elapsed().as_micros() ); let mut writer = self.writer.lock().unwrap(); writer .write_all(json.as_bytes()) .expect("Error writing tracing"); } fn log_event(&self, name: &str, ph: &str) { let json = format!( "{{ \"name\": \"{}\", \"cat\": \"PERF\", \"ph\": \"{ph}\", \"pid\": 1, \"tid\": {}, \"ts\": {} }},\n", name, get_thread_id(), self.now.elapsed().as_micros() ); let mut writer = self.writer.lock().unwrap(); writer .write_all(json.as_bytes()) .expect("Error writing tracing"); } } pub fn get_thread_id() -> u64 { let thread_id = std::thread::current().id(); format!("{thread_id:?}") .trim_start_matches("ThreadId(") .trim_end_matches(")") .parse() .expect("Error parsing thread id") } pub fn trace_begin(name: &str) { RECORDER.begin_task(name); } pub fn trace_end(name: &str) { RECORDER.end_task(name); } pub fn log_memory_usage(name: &str, value: u64) { RECORDER.log_memory_usage(name, value); } pub fn with_trace(label: &str, tracing_enabled: bool, f: F) -> T where F: FnOnce() -> T, { if tracing_enabled { trace_begin(label); } let result = f(); if tracing_enabled { trace_end(label); } result } pub async fn with_trace_async(label: &str, tracing_enabled: bool, f: F) -> T where F: FnOnce() -> Fut, Fut: std::future::Future, { if tracing_enabled { trace_begin(label); } let result = f().await; if tracing_enabled { trace_end(label); } result } ================================================ FILE: native/common/src/utils.rs ================================================ // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. /// Converts a slice of bytes to i128. The bytes are serialized in big-endian order by /// `BigInteger.toByteArray()` in Java. pub fn bytes_to_i128(slice: &[u8]) -> i128 { let mut bytes = [0; 16]; let mut i = 0; while i != 16 && i != slice.len() { bytes[i] = slice[slice.len() - 1 - i]; i += 1; } // if the decimal is negative, we need to flip all the bits if (slice[0] as i8) < 0 { while i < 16 { bytes[i] = !bytes[i]; i += 1; } } i128::from_le_bytes(bytes) } ================================================ FILE: native/core/Cargo.toml ================================================ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. [package] name = "datafusion-comet" version = { workspace = true } homepage = "https://datafusion.apache.org/comet" repository = "https://github.com/apache/datafusion-comet" authors = ["Apache DataFusion "] description = "Apache DataFusion Comet: High performance accelerator for Apache Spark" readme = "README.md" license = "Apache-2.0" edition = "2021" include = [ "benches/*.rs", "src/**/*.rs", "Cargo.toml", ] # this crate is used in the Spark plugin and does not contain public Rust APIs so we do not publish this crate publish = false [dependencies] arrow = { workspace = true } parquet = { workspace = true, default-features = false, features = ["experimental", "arrow"] } futures = { workspace = true } mimalloc = { version = "*", default-features = false, optional = true } tikv-jemallocator = { version = "0.6.1", optional = true, features = ["disable_initial_exec_tls"] } tikv-jemalloc-ctl = { version = "0.6.1", optional = true, features = ["disable_initial_exec_tls", "stats"] } tokio = { version = "1", features = ["rt-multi-thread"] } async-trait = { workspace = true } log = "0.4" log4rs = "1.4.0" prost = "0.14.3" jni = "0.22.4" rand = { workspace = true } num = { workspace = true } bytes = { workspace = true } tempfile = "3.26.0" itertools = "0.14.0" paste = "1.0.14" datafusion = { workspace = true, features = ["parquet_encryption", "sql"] } datafusion-physical-expr-adapter = { workspace = true } datafusion-datasource = { workspace = true } datafusion-spark = { workspace = true } once_cell = "1.18.0" datafusion-comet-common = { workspace = true } datafusion-comet-spark-expr = { workspace = true } datafusion-comet-jni-bridge = { workspace = true } datafusion-comet-proto = { workspace = true } datafusion-comet-shuffle = { workspace = true } object_store = { workspace = true } url = { workspace = true } aws-config = { workspace = true } aws-credential-types = { workspace = true } parking_lot = "0.12.5" datafusion-comet-objectstore-hdfs = { path = "../hdfs", optional = true, default-features = false, features = ["hdfs"] } reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-native-roots", "http2"] } object_store_opendal = { git = "https://github.com/apache/opendal", rev = "6909efcdfd12b3b2ac3a76f654c35ee576811512", package = "object_store_opendal", optional = true} hdfs-sys = {version = "0.3", optional = true, features = ["hdfs_3_3"]} opendal = { git = "https://github.com/apache/opendal", rev = "6909efcdfd12b3b2ac3a76f654c35ee576811512", optional = true, features = ["services-hdfs"] } iceberg = { workspace = true } iceberg-storage-opendal = { workspace = true } serde_json = "1.0" uuid = "1.23.0" [target.'cfg(target_os = "linux")'.dependencies] procfs = "0.18.0" [target.'cfg(target_os = "macos")'.dependencies] hdrs = { version = "0.3.2", features = ["vendored"] } [dev-dependencies] pprof = { version = "0.15", features = ["flamegraph"] } criterion = { version = "0.7", features = ["async", "async_tokio", "async_std"] } jni = { version = "0.22.4", features = ["invocation"] } lazy_static = "1.4" assertables = "9" hex = "0.4.3" datafusion-functions-nested = { version = "53.1.0" } [features] backtrace = ["datafusion/backtrace"] default = ["hdfs-opendal"] hdfs = ["datafusion-comet-objectstore-hdfs"] hdfs-opendal = ["opendal", "object_store_opendal", "hdfs-sys"] jemalloc = ["tikv-jemallocator", "tikv-jemalloc-ctl"] # exclude optional packages from cargo machete verifications [package.metadata.cargo-machete] ignored = ["hdfs-sys", "paste"] [lib] name = "comet" # "rlib" is for benchmarking with criterion. crate-type = ["cdylib", "rlib"] [[bench]] name = "parquet_read" harness = false [[bench]] name = "bit_util" harness = false [[bench]] name = "parquet_decode" harness = false [[bench]] name = "array_element_append" harness = false ================================================ FILE: native/core/benches/array_element_append.rs ================================================ // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. //! Micro-benchmarks for SparkUnsafeArray element iteration. //! //! This tests the low-level `append_to_builder` function which converts //! SparkUnsafeArray elements to Arrow array builders. This is the inner loop //! used when processing List/Array columns in JVM shuffle. use arrow::array::builder::{ Date32Builder, Float64Builder, Int32Builder, Int64Builder, TimestampMicrosecondBuilder, }; use arrow::datatypes::{DataType, TimeUnit}; use comet::execution::shuffle::spark_unsafe::list::{append_to_builder, SparkUnsafeArray}; use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; const NUM_ELEMENTS: usize = 10000; /// Create a SparkUnsafeArray in memory with i32 elements. /// Layout: /// - 8 bytes: num_elements (i64) /// - null bitset: 8 bytes per 64 elements /// - element data: 4 bytes per element (i32) fn create_spark_unsafe_array_i32(num_elements: usize, with_nulls: bool) -> Vec { // Header size: 8 (num_elements) + ceil(num_elements/64) * 8 (null bitset) let null_bitset_words = num_elements.div_ceil(64); let header_size = 8 + null_bitset_words * 8; let data_size = num_elements * 4; // i32 = 4 bytes let total_size = header_size + data_size; let mut buffer = vec![0u8; total_size]; // Write num_elements buffer[0..8].copy_from_slice(&(num_elements as i64).to_le_bytes()); // Write null bitset (set every 10th element as null if with_nulls) if with_nulls { for i in (0..num_elements).step_by(10) { let word_idx = i / 64; let bit_idx = i % 64; let word_offset = 8 + word_idx * 8; let current_word = i64::from_le_bytes(buffer[word_offset..word_offset + 8].try_into().unwrap()); let new_word = current_word | (1i64 << bit_idx); buffer[word_offset..word_offset + 8].copy_from_slice(&new_word.to_le_bytes()); } } // Write element data for i in 0..num_elements { let offset = header_size + i * 4; buffer[offset..offset + 4].copy_from_slice(&(i as i32).to_le_bytes()); } buffer } /// Create a SparkUnsafeArray in memory with i64 elements. fn create_spark_unsafe_array_i64(num_elements: usize, with_nulls: bool) -> Vec { let null_bitset_words = num_elements.div_ceil(64); let header_size = 8 + null_bitset_words * 8; let data_size = num_elements * 8; // i64 = 8 bytes let total_size = header_size + data_size; let mut buffer = vec![0u8; total_size]; // Write num_elements buffer[0..8].copy_from_slice(&(num_elements as i64).to_le_bytes()); // Write null bitset if with_nulls { for i in (0..num_elements).step_by(10) { let word_idx = i / 64; let bit_idx = i % 64; let word_offset = 8 + word_idx * 8; let current_word = i64::from_le_bytes(buffer[word_offset..word_offset + 8].try_into().unwrap()); let new_word = current_word | (1i64 << bit_idx); buffer[word_offset..word_offset + 8].copy_from_slice(&new_word.to_le_bytes()); } } // Write element data for i in 0..num_elements { let offset = header_size + i * 8; buffer[offset..offset + 8].copy_from_slice(&(i as i64).to_le_bytes()); } buffer } /// Create a SparkUnsafeArray in memory with f64 elements. fn create_spark_unsafe_array_f64(num_elements: usize, with_nulls: bool) -> Vec { let null_bitset_words = num_elements.div_ceil(64); let header_size = 8 + null_bitset_words * 8; let data_size = num_elements * 8; // f64 = 8 bytes let total_size = header_size + data_size; let mut buffer = vec![0u8; total_size]; // Write num_elements buffer[0..8].copy_from_slice(&(num_elements as i64).to_le_bytes()); // Write null bitset if with_nulls { for i in (0..num_elements).step_by(10) { let word_idx = i / 64; let bit_idx = i % 64; let word_offset = 8 + word_idx * 8; let current_word = i64::from_le_bytes(buffer[word_offset..word_offset + 8].try_into().unwrap()); let new_word = current_word | (1i64 << bit_idx); buffer[word_offset..word_offset + 8].copy_from_slice(&new_word.to_le_bytes()); } } // Write element data for i in 0..num_elements { let offset = header_size + i * 8; buffer[offset..offset + 8].copy_from_slice(&(i as f64).to_le_bytes()); } buffer } fn benchmark_array_conversion(c: &mut Criterion) { let mut group = c.benchmark_group("spark_unsafe_array_to_arrow"); // Benchmark i32 array conversion for with_nulls in [false, true] { let buffer = create_spark_unsafe_array_i32(NUM_ELEMENTS, with_nulls); let array = SparkUnsafeArray::new(buffer.as_ptr() as i64); let null_str = if with_nulls { "with_nulls" } else { "no_nulls" }; group.bench_with_input( BenchmarkId::new("i32", null_str), &(&array, &buffer), |b, (array, _buffer)| { b.iter(|| { let mut builder = Int32Builder::with_capacity(NUM_ELEMENTS); if with_nulls { append_to_builder::(&DataType::Int32, &mut builder, array).unwrap(); } else { append_to_builder::(&DataType::Int32, &mut builder, array).unwrap(); } builder.finish() }); }, ); } // Benchmark i64 array conversion for with_nulls in [false, true] { let buffer = create_spark_unsafe_array_i64(NUM_ELEMENTS, with_nulls); let array = SparkUnsafeArray::new(buffer.as_ptr() as i64); let null_str = if with_nulls { "with_nulls" } else { "no_nulls" }; group.bench_with_input( BenchmarkId::new("i64", null_str), &(&array, &buffer), |b, (array, _buffer)| { b.iter(|| { let mut builder = Int64Builder::with_capacity(NUM_ELEMENTS); if with_nulls { append_to_builder::(&DataType::Int64, &mut builder, array).unwrap(); } else { append_to_builder::(&DataType::Int64, &mut builder, array).unwrap(); } builder.finish() }); }, ); } // Benchmark f64 array conversion for with_nulls in [false, true] { let buffer = create_spark_unsafe_array_f64(NUM_ELEMENTS, with_nulls); let array = SparkUnsafeArray::new(buffer.as_ptr() as i64); let null_str = if with_nulls { "with_nulls" } else { "no_nulls" }; group.bench_with_input( BenchmarkId::new("f64", null_str), &(&array, &buffer), |b, (array, _buffer)| { b.iter(|| { let mut builder = Float64Builder::with_capacity(NUM_ELEMENTS); if with_nulls { append_to_builder::(&DataType::Float64, &mut builder, array).unwrap(); } else { append_to_builder::(&DataType::Float64, &mut builder, array) .unwrap(); } builder.finish() }); }, ); } // Benchmark date32 array conversion (same memory layout as i32) for with_nulls in [false, true] { let buffer = create_spark_unsafe_array_i32(NUM_ELEMENTS, with_nulls); let array = SparkUnsafeArray::new(buffer.as_ptr() as i64); let null_str = if with_nulls { "with_nulls" } else { "no_nulls" }; group.bench_with_input( BenchmarkId::new("date32", null_str), &(&array, &buffer), |b, (array, _buffer)| { b.iter(|| { let mut builder = Date32Builder::with_capacity(NUM_ELEMENTS); if with_nulls { append_to_builder::(&DataType::Date32, &mut builder, array).unwrap(); } else { append_to_builder::(&DataType::Date32, &mut builder, array).unwrap(); } builder.finish() }); }, ); } // Benchmark timestamp array conversion (same memory layout as i64) for with_nulls in [false, true] { let buffer = create_spark_unsafe_array_i64(NUM_ELEMENTS, with_nulls); let array = SparkUnsafeArray::new(buffer.as_ptr() as i64); let null_str = if with_nulls { "with_nulls" } else { "no_nulls" }; group.bench_with_input( BenchmarkId::new("timestamp", null_str), &(&array, &buffer), |b, (array, _buffer)| { b.iter(|| { let mut builder = TimestampMicrosecondBuilder::with_capacity(NUM_ELEMENTS); let dt = DataType::Timestamp(TimeUnit::Microsecond, None); if with_nulls { append_to_builder::(&dt, &mut builder, array).unwrap(); } else { append_to_builder::(&dt, &mut builder, array).unwrap(); } builder.finish() }); }, ); } group.finish(); } fn config() -> Criterion { Criterion::default() } criterion_group! { name = benches; config = config(); targets = benchmark_array_conversion } criterion_main!(benches); ================================================ FILE: native/core/benches/bit_util.rs ================================================ // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. use std::{mem::size_of, time::Duration}; use rand::{rng, RngExt}; use arrow::buffer::Buffer; use comet::common::bit::{ log2, read_num_bytes_u32, read_num_bytes_u64, read_u32, read_u64, set_bits, trailing_bits, BitReader, BitWriter, }; use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; use std::hint::black_box; /// Benchmark to measure bit_util performance. /// To run this benchmark: /// `cd core && cargo bench --bench bit_util` /// Results will be written to "core/target/criterion/bit_util/" fn criterion_benchmark(c: &mut Criterion) { let mut group = c.benchmark_group("bit_util"); const N: usize = 1024 * 1024; let mut writer: BitWriter = BitWriter::new(N * 10); for _ in 0..N { if !writer.put_vlq_int(rng().random::()) { break; } } let buffer = writer.consume(); let buffer = Buffer::from(buffer.as_slice()); // log2 for bits in (0..64).step_by(3) { let x = 1u64 << bits; group.bench_with_input(BenchmarkId::new("log2", bits), &x, |b, &x| { b.iter(|| log2(black_box(x))); }); } // set_bits for offset in (0..16).step_by(3) { for length in (0..16).step_by(3) { let x = (offset, length); group.bench_with_input( BenchmarkId::new("set_bits", format!("offset_{}_length_{}", x.0, x.1)), &x, |b, &x| { b.iter(|| set_bits(&mut [0u8; 4], black_box(x.0), black_box(x.1))); }, ); } } // get_vlq_int group.bench_function("get_vlq_int", |b| { b.iter(|| { let mut reader: BitReader = BitReader::new_all(buffer.slice(0)); bench_get_vlq_int(&mut reader) }) }); // get_bits for offset in (0..32).step_by(17) { for num_bits in (1..5).step_by(1) { let x = (offset, num_bits); group.bench_with_input( BenchmarkId::new("get_bits", format!("offset_{}_num_bits_{}", x.0, x.1)), &x, |b, &x| { let mut reader: BitReader = BitReader::new_all(buffer.slice(0)); b.iter(|| reader.get_bits(&mut [0u8; 4], black_box(x.0), black_box(x.1))); }, ); } } // get_aligned for num_bytes in (1..=size_of::()).step_by(3) { let x = num_bytes; group.bench_with_input( BenchmarkId::new("get_aligned", format!("u8_num_bytes_{x}")), &x, |b, &x| { let mut reader: BitReader = BitReader::new_all(buffer.slice(0)); b.iter(|| reader.get_aligned::(black_box(x))); }, ); } for num_bytes in (1..=size_of::()).step_by(3) { let x = num_bytes; group.bench_with_input( BenchmarkId::new("get_aligned", format!("u32_num_bytes_{x}")), &x, |b, &x| { let mut reader: BitReader = BitReader::new_all(buffer.slice(0)); b.iter(|| reader.get_aligned::(black_box(x))); }, ); } for num_bytes in (1..=size_of::()).step_by(3) { let x = num_bytes; group.bench_with_input( BenchmarkId::new("get_aligned", format!("i32_num_bytes_{x}")), &x, |b, &x| { let mut reader: BitReader = BitReader::new_all(buffer.slice(0)); b.iter(|| reader.get_aligned::(black_box(x))); }, ); } // get_value for num_bytes in (1..=size_of::()).step_by(3) { let x = num_bytes * 8; group.bench_with_input( BenchmarkId::new("get_value", format!("i32_num_bits_{x}")), &x, |b, &x| { let mut reader: BitReader = BitReader::new_all(buffer.slice(0)); b.iter(|| reader.get_value::(black_box(x))); }, ); } // read_num_bytes_u64 for num_bytes in (1..=8).step_by(7) { let x = num_bytes; group.bench_with_input( BenchmarkId::new("read_num_bytes_u64", format!("num_bytes_{x}")), &x, |b, &x| { b.iter(|| read_num_bytes_u64(black_box(x), black_box(buffer.as_slice()))); }, ); } // read_num_bytes_u32 for num_bytes in (1..=4).step_by(3) { let x = num_bytes; group.bench_with_input( BenchmarkId::new("read_num_bytes_u32", format!("num_bytes_{x}")), &x, |b, &x| { b.iter(|| read_num_bytes_u32(black_box(x), black_box(buffer.as_slice()))); }, ); } // trailing_bits for length in (0..=64).step_by(32) { let x = length; group.bench_with_input( BenchmarkId::new("trailing_bits", format!("num_bits_{x}")), &x, |b, &x| { b.iter(|| trailing_bits(black_box(1234567890), black_box(x))); }, ); } // read_u64 group.bench_function("read_u64", |b| { b.iter(|| read_u64(black_box(&[0u8; 8]))); }); // read_u32 group.bench_function("read_u32", |b| { b.iter(|| read_u32(black_box(&[0u8; 4]))); }); // get_u32_value group.bench_function("get_u32_value", |b| { b.iter(|| { let mut reader: BitReader = BitReader::new_all(buffer.slice(0)); for _ in 0..(buffer.len() * 8 / 31) { black_box(reader.get_u32_value(black_box(31))); } }) }); group.finish(); } fn bench_get_vlq_int(reader: &mut BitReader) { while let Some(v) = reader.get_vlq_int() { black_box(v); } } fn config() -> Criterion { Criterion::default() .measurement_time(Duration::from_millis(500)) .warm_up_time(Duration::from_millis(500)) } criterion_group! { name = benches; config = config(); targets = criterion_benchmark } criterion_main!(benches); ================================================ FILE: native/core/benches/common.rs ================================================ // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. use arrow::error::ArrowError; use arrow::{ array::{DictionaryArray, Int64Array, PrimitiveArray}, datatypes::{ArrowPrimitiveType, Int32Type}, }; use rand::{ distr::{Distribution, StandardUniform}, rngs::StdRng, RngExt, SeedableRng, }; use std::sync::Arc; /// Returns fixed seedable RNG pub fn seedable_rng() -> StdRng { StdRng::seed_from_u64(42) } pub fn create_int64_array(size: usize, null_density: f32, min: i64, max: i64) -> Int64Array { let mut rng = seedable_rng(); (0..size) .map(|_| { if rng.random::() < null_density { None } else { Some(rng.random_range(min..max)) } }) .collect() } #[allow(dead_code)] pub fn create_primitive_array(size: usize, null_density: f32) -> PrimitiveArray where T: ArrowPrimitiveType, StandardUniform: Distribution, { let mut rng = seedable_rng(); (0..size) .map(|_| { if rng.random::() < null_density { None } else { Some(rng.random()) } }) .collect() } /// Creates a dictionary with random keys and values, with value type `T`. /// Note here the keys are the dictionary indices. #[allow(dead_code)] pub fn create_dictionary_array( size: usize, value_size: usize, null_density: f32, ) -> Result, ArrowError> where T: ArrowPrimitiveType, StandardUniform: Distribution, { // values are not null let values = create_primitive_array::(value_size, 0.0); let keys = create_primitive_array::(size, null_density) .iter() .map(|v| v.map(|w| w.abs() % (value_size as i32))) .collect(); DictionaryArray::try_new(keys, Arc::new(values)) } ================================================ FILE: native/core/benches/parquet_decode.rs ================================================ // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. use arrow::datatypes::ToByteSlice; use comet::parquet::read::values::{copy_i32_to_i16, copy_i32_to_u16, copy_i64_to_i64}; use criterion::{criterion_group, criterion_main, Criterion}; fn criterion_benchmark(c: &mut Criterion) { let num = 1000; let source = vec![78_i8; num * 8]; let mut group = c.benchmark_group("parquet_decode"); group.bench_function("decode_i32_to_i16", |b| { let mut dest: Vec = vec![b' '; num * 2]; b.iter(|| { copy_i32_to_i16(source.to_byte_slice(), dest.as_mut_slice(), num); }); }); group.bench_function("decode_i32_to_u16", |b| { let mut dest: Vec = vec![b' '; num * 4]; b.iter(|| { copy_i32_to_u16(source.to_byte_slice(), dest.as_mut_slice(), num); }); }); group.bench_function("decode_i64_to_i64", |b| { let mut dest: Vec = vec![b' '; num * 8]; b.iter(|| { copy_i64_to_i64(source.to_byte_slice(), dest.as_mut_slice(), num); }); }); } // Create UTF8 batch with strings representing ints, floats, nulls fn config() -> Criterion { Criterion::default() } criterion_group! { name = benches; config = config(); targets = criterion_benchmark } criterion_main!(benches); ================================================ FILE: native/core/benches/parquet_read.rs ================================================ // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. mod perf; use std::sync::Arc; use arrow::{array::ArrayData, buffer::Buffer}; use comet::parquet::{read::ColumnReader, util::jni::TypePromotionInfo}; use criterion::{criterion_group, criterion_main, Criterion}; use parquet::{ basic::{Encoding, Type as PhysicalType}, column::page::{PageIterator, PageReader}, data_type::Int32Type, schema::types::{ ColumnDescPtr, ColumnDescriptor, ColumnPath, PrimitiveTypeBuilder, SchemaDescPtr, TypePtr, }, }; use comet::parquet::util::test_common::page_util::{ DataPageBuilder, DataPageBuilderImpl, InMemoryPageIterator, }; use perf::FlamegraphProfiler; use rand::{prelude::StdRng, RngExt, SeedableRng}; fn bench(c: &mut Criterion) { let expected_num_values: usize = NUM_PAGES * VALUES_PER_PAGE; let mut group = c.benchmark_group("comet_parquet_read"); let schema = build_test_schema(); let pages = build_plain_int32_pages(schema.column(0), 0.0); group.bench_function("INT/PLAIN/NOT_NULL", |b| { let t = TypePtr::new( PrimitiveTypeBuilder::new("f", PhysicalType::INT32) .with_length(4) .build() .unwrap(), ); b.iter(|| { let cd = ColumnDescriptor::new(t.clone(), 0, 0, ColumnPath::from(Vec::new())); let promotion_info = TypePromotionInfo::new(PhysicalType::INT32, -1, -1, 32); let mut column_reader = TestColumnReader::new( cd, promotion_info, BATCH_SIZE, pages.clone(), expected_num_values, ); let mut total = 0; for batch in column_reader.by_ref() { total += batch.len(); ::std::mem::forget(batch); } assert_eq!(total, expected_num_values); }); }); } fn profiled() -> Criterion { Criterion::default().with_profiler(FlamegraphProfiler::new(100)) } criterion_group! { name = benches; config = profiled(); targets = bench } criterion_main!(benches); fn build_test_schema() -> SchemaDescPtr { use parquet::schema::{parser::parse_message_type, types::SchemaDescriptor}; let message_type = " message test_schema { REQUIRED INT32 c1; OPTIONAL INT32 c2; } "; parse_message_type(message_type) .map(|t| Arc::new(SchemaDescriptor::new(Arc::new(t)))) .unwrap() } fn seedable_rng() -> StdRng { StdRng::seed_from_u64(42) } // test data params const NUM_PAGES: usize = 1000; const VALUES_PER_PAGE: usize = 10_000; const BATCH_SIZE: usize = 4096; fn build_plain_int32_pages( column_desc: ColumnDescPtr, null_density: f32, ) -> impl PageIterator + Clone { let max_def_level = column_desc.max_def_level(); let max_rep_level = column_desc.max_rep_level(); let rep_levels = vec![0; VALUES_PER_PAGE]; let mut rng = seedable_rng(); let mut pages: Vec = Vec::new(); let mut int32_value = 0; for _ in 0..NUM_PAGES { // generate page let mut values = Vec::with_capacity(VALUES_PER_PAGE); let mut def_levels = Vec::with_capacity(VALUES_PER_PAGE); for _ in 0..VALUES_PER_PAGE { let def_level = if rng.random::() < null_density { max_def_level - 1 } else { max_def_level }; if def_level == max_def_level { int32_value += 1; values.push(int32_value); } def_levels.push(def_level); } let mut page_builder = DataPageBuilderImpl::new(column_desc.clone(), values.len() as u32, true); page_builder.add_rep_levels(max_rep_level, &rep_levels); page_builder.add_def_levels(max_def_level, &def_levels); page_builder.add_values::(Encoding::PLAIN, &values); pages.push(page_builder.consume()); } // Since `InMemoryPageReader` is not exposed from parquet crate, here we use // `InMemoryPageIterator` instead which is a Iter>. InMemoryPageIterator::new(vec![pages]) } struct TestColumnReader { inner: ColumnReader, pages: Box, batch_size: usize, total_num_values: usize, total_num_values_read: usize, first_page_loaded: bool, } impl TestColumnReader { pub fn new( cd: ColumnDescriptor, promotion_info: TypePromotionInfo, batch_size: usize, mut page_iter: impl PageIterator + 'static, total_num_values: usize, ) -> Self { let reader = ColumnReader::get(cd, promotion_info, batch_size, false, false); let first = page_iter.next().unwrap().unwrap(); Self { inner: reader, pages: first, batch_size, total_num_values, total_num_values_read: 0, first_page_loaded: false, } } fn load_page(&mut self) { if let Some(page) = self.pages.get_next_page().unwrap() { let num_values = page.num_values() as usize; let buffer = Buffer::from_slice_ref(page.buffer()); self.inner.set_page_v1(num_values, buffer, page.encoding()); } } } impl Iterator for TestColumnReader { type Item = ArrayData; fn next(&mut self) -> Option { if self.total_num_values_read >= self.total_num_values { return None; } if !self.first_page_loaded { self.load_page(); self.first_page_loaded = true; } self.inner.reset_batch(); let total = ::std::cmp::min( self.batch_size, self.total_num_values - self.total_num_values_read, ); let mut left = total; while left > 0 { let (num_read, _) = self.inner.read_batch(left, 0); if num_read < left { self.load_page(); } left -= num_read; } self.total_num_values_read += total; Some(self.inner.current_batch().unwrap()) } } ================================================ FILE: native/core/benches/perf.rs ================================================ // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. use std::{fs::File, os::raw::c_int, path::Path}; use criterion::profiler::Profiler; use pprof::ProfilerGuard; /// A custom profiler for criterion which generates flamegraph. /// /// Mostly followed this blog post: https://www.jibbow.com/posts/criterion-flamegraphs/ /// After `cargo bench --bench -- --profile-time=