Repository: secretflow/scql Branch: main Commit: 8107675dd7e7 Files: 849 Total size: 10.6 MB Directory structure: gitextract_kxn6szmx/ ├── .bazeliskrc ├── .bazelrc ├── .circleci/ │ ├── config.yml │ ├── coverage-config.yml │ ├── diff-cover-config.yml │ ├── full-unittest-config.yml │ ├── lite-unittest-config.yml │ └── path-filtering/ │ └── unittest.conf ├── .clang-format ├── .clang-tidy ├── .coveralls.yml ├── .devcontainer/ │ ├── Dockerfile │ └── devcontainer.json ├── .github/ │ ├── CODEOWNERS │ ├── ISSUE_TEMPLATE/ │ │ ├── Consulting_issue_template.yaml │ │ └── Error_Issue_Template.yaml │ └── workflows/ │ ├── black.yml │ ├── buildifier.yml │ ├── cla.yml │ ├── clang-format-linter.yml │ ├── codeql.yml │ ├── dependency-review.yml │ ├── docs-check.yml │ ├── docs-publish.yml │ ├── golangci-lint.yml │ ├── govulncheck.yml │ ├── license-check.yml │ ├── oscp.yml │ ├── scorecards.yml │ ├── stale.yml │ ├── trigger-ci-cov.yml │ ├── trigger-ci-full-ut.yml │ ├── trigger-ci-lite-ut.yml │ ├── trigger-diff-coverage.yml │ ├── whitespace-check.yml │ └── yaml-linter.yml ├── .gitignore ├── .golangci.yml ├── .licenserc.yaml ├── .markdownlint.yaml ├── .pre-commit-config.yaml ├── .vscode/ │ ├── cspell.json │ ├── extensions.json │ └── settings.json ├── BUILD.bazel ├── CHANGELOG.md ├── CONTRIBUTING.md ├── LEGAL.md ├── LICENSE ├── MODULE.bazel ├── Makefile ├── README.md ├── REPO_LAYOUT.md ├── SECURITY.md ├── api/ │ ├── BUILD.bazel │ ├── buf.yaml │ ├── common.proto │ ├── core.proto │ ├── engine.proto │ ├── generate_proto.sh │ ├── interpreter.proto │ ├── scql_task.proto │ ├── status.proto │ ├── status_code.proto │ ├── subgraph.proto │ ├── v1/ │ │ ├── BUILD.bazel │ │ ├── column.proto │ │ └── genproto.sh │ └── v1alpha1/ │ ├── BUILD.bazel │ └── compiler.proto ├── bazel/ │ ├── BUILD.bazel │ ├── defs.bzl │ ├── patches/ │ │ ├── BUILD.bazel │ │ ├── grpc-plugin.patch │ │ ├── protobuf-xla.patch │ │ └── rules_foreign_cc.patch │ └── repositories.bzl ├── cmd/ │ └── docgen/ │ ├── main.go │ └── scql_operators.md.tmpl ├── docs/ │ ├── CONTRIBUTING.md │ ├── Makefile │ ├── _static/ │ │ ├── css/ │ │ │ └── custom.css │ │ └── js/ │ │ └── custom.js │ ├── conf.py │ ├── index.rst │ ├── intro/ │ │ ├── index.rst │ │ └── opencore-quickstart.rst │ ├── locales/ │ │ └── zh_CN/ │ │ └── LC_MESSAGES/ │ │ ├── index.po │ │ ├── intro/ │ │ │ ├── index.po │ │ │ └── opencore-quickstart.po │ │ ├── reference/ │ │ │ ├── compiler-config.po │ │ │ ├── engine-config.po │ │ │ ├── implementation-status.po │ │ │ ├── index.po │ │ │ ├── lang/ │ │ │ │ ├── manual.po │ │ │ │ └── mysql-compatibility.po │ │ │ └── operators.po │ │ └── topics/ │ │ ├── faq.po │ │ ├── index.po │ │ ├── security/ │ │ │ └── overview.po │ │ └── system/ │ │ └── intro.po │ ├── reference/ │ │ ├── compiler-config.rst │ │ ├── engine-config.rst │ │ ├── implementation-status.rst │ │ ├── index.rst │ │ ├── lang/ │ │ │ ├── manual.rst │ │ │ └── mysql-compatibility.rst │ │ └── operators.md │ ├── requirements.txt │ └── topics/ │ ├── faq.rst │ ├── index.rst │ ├── security/ │ │ └── overview.rst │ └── system/ │ └── intro.rst ├── engine/ │ ├── auth/ │ │ ├── BUILD.bazel │ │ ├── authenticator.cc │ │ ├── authenticator.h │ │ ├── authenticator_test.cc │ │ ├── authorized_profile.cc │ │ ├── authorized_profile.h │ │ └── authorized_profile.proto │ ├── bazel/ │ │ ├── BUILD.bazel │ │ ├── duckdb.BUILD │ │ ├── engine_deps.bzl │ │ ├── gperftools.BUILD │ │ ├── mysql.BUILD │ │ ├── patches/ │ │ │ ├── dataproxy.patch │ │ │ ├── duckdb.patch │ │ │ ├── mysql.patch │ │ │ ├── poco.patch │ │ │ └── psi.patch │ │ ├── perfetto.BUILD │ │ ├── poco.BUILD │ │ ├── postgres.BUILD │ │ ├── scql.bzl │ │ └── sqlite3.BUILD │ ├── core/ │ │ ├── BUILD.bazel │ │ ├── arrow_helper.h │ │ ├── primitive_builder.cc │ │ ├── primitive_builder.h │ │ ├── primitive_builder_test.cc │ │ ├── string_tensor_builder.cc │ │ ├── string_tensor_builder.h │ │ ├── string_tensor_builder_test.cc │ │ ├── tensor.cc │ │ ├── tensor.h │ │ ├── tensor_batch_reader.cc │ │ ├── tensor_batch_reader.h │ │ ├── tensor_batch_reader_test.cc │ │ ├── tensor_builder.cc │ │ ├── tensor_builder.h │ │ ├── tensor_constructor.cc │ │ ├── tensor_constructor.h │ │ ├── tensor_constructor_test.cc │ │ ├── tensor_slice.cc │ │ ├── tensor_slice.h │ │ ├── type.cc │ │ └── type.h │ ├── datasource/ │ │ ├── BUILD.bazel │ │ ├── arrow_sql_adaptor.cc │ │ ├── arrow_sql_adaptor.h │ │ ├── arrow_sql_adaptor_factory.h │ │ ├── arrow_sql_adaptor_test.cc │ │ ├── csvdb_adaptor.cc │ │ ├── csvdb_adaptor.h │ │ ├── csvdb_adaptor_factory.cc │ │ ├── csvdb_adaptor_factory.h │ │ ├── csvdb_adaptor_test.cc │ │ ├── csvdb_conf.proto │ │ ├── dataproxy_conf.proto │ │ ├── datasource.proto │ │ ├── datasource_adaptor.cc │ │ ├── datasource_adaptor.h │ │ ├── datasource_adaptor_factory.h │ │ ├── datasource_adaptor_mgr.cc │ │ ├── datasource_adaptor_mgr.h │ │ ├── dm_adaptor.cc │ │ ├── dm_adaptor.h │ │ ├── dm_adaptor_factory.cc │ │ ├── dm_adaptor_factory.h │ │ ├── dm_adaptor_kuscia_test.cc │ │ ├── dp_adaptor.cc │ │ ├── dp_adaptor.h │ │ ├── dp_adaptor_factory.cc │ │ ├── dp_adaptor_factory.h │ │ ├── dp_adaptor_test.cc │ │ ├── duckdb_wrapper.cc │ │ ├── duckdb_wrapper.h │ │ ├── duckdb_wrapper_test.cc │ │ ├── embed_router.cc │ │ ├── embed_router.h │ │ ├── embed_router.proto │ │ ├── embed_router_test.cc │ │ ├── http_router.cc │ │ ├── http_router.h │ │ ├── http_router.proto │ │ ├── kuscia_datamesh_router.cc │ │ ├── kuscia_datamesh_router.h │ │ ├── kuscia_datamesh_router_test.cc │ │ ├── mock_router_readme.md │ │ ├── mock_router_server.py │ │ ├── odbc_adaptor.cc │ │ ├── odbc_adaptor.h │ │ ├── odbc_adaptor_factory.cc │ │ ├── odbc_adaptor_factory.h │ │ ├── odbc_adaptor_mysql_test.cc │ │ ├── odbc_adaptor_sqlite_test.cc │ │ ├── odbc_connector.cc │ │ ├── odbc_connector.h │ │ ├── requirements.txt │ │ ├── router.h │ │ └── run_odbc_adaptor_mysql_test.sh │ ├── exe/ │ │ ├── BUILD.bazel │ │ ├── flags.cc │ │ ├── flags.h │ │ ├── main.cc │ │ └── version.h │ ├── framework/ │ │ ├── BUILD.bazel │ │ ├── exec.cc │ │ ├── exec.h │ │ ├── executor.cc │ │ ├── executor.h │ │ ├── operator.cc │ │ ├── operator.h │ │ ├── party_info.cc │ │ ├── party_info.h │ │ ├── party_info_test.cc │ │ ├── registry.cc │ │ ├── registry.h │ │ ├── session.cc │ │ ├── session.h │ │ ├── session_manager.cc │ │ ├── session_manager.h │ │ ├── session_manager_test.cc │ │ ├── tensor_table.cc │ │ └── tensor_table.h │ ├── link/ │ │ ├── BUILD.bazel │ │ ├── channel_manager.cc │ │ ├── channel_manager.h │ │ ├── listener.cc │ │ ├── listener.h │ │ ├── listener_test.cc │ │ ├── mux_link_factory.cc │ │ ├── mux_link_factory.h │ │ ├── mux_receiver.proto │ │ ├── mux_receiver_service.cc │ │ ├── mux_receiver_service.h │ │ ├── mux_receiver_service_test.cc │ │ ├── rpc_helper.cc │ │ ├── rpc_helper.h │ │ └── rpc_helper_test.cc │ ├── operator/ │ │ ├── BUILD.bazel │ │ ├── all_ops_register.cc │ │ ├── all_ops_register.h │ │ ├── all_ops_register_test.cc │ │ ├── arithmetic.cc │ │ ├── arithmetic.h │ │ ├── arithmetic_test.cc │ │ ├── arrow_func.cc │ │ ├── arrow_func.h │ │ ├── arrow_func_test.cc │ │ ├── binary_base.cc │ │ ├── binary_base.h │ │ ├── binary_test.h │ │ ├── broadcast_to.cc │ │ ├── broadcast_to.h │ │ ├── broadcast_to_test.cc │ │ ├── bucket.cc │ │ ├── bucket.h │ │ ├── bucket_bench.cc │ │ ├── bucket_test.cc │ │ ├── case_when.cc │ │ ├── case_when.h │ │ ├── case_when_test.cc │ │ ├── cast.cc │ │ ├── cast.h │ │ ├── cast_test.cc │ │ ├── coalesce.cc │ │ ├── coalesce.h │ │ ├── coalesce_test.cc │ │ ├── compare.cc │ │ ├── compare.h │ │ ├── compare_test.cc │ │ ├── concat.cc │ │ ├── concat.h │ │ ├── concat_test.cc │ │ ├── constant.cc │ │ ├── constant.h │ │ ├── constant_test.cc │ │ ├── copy.cc │ │ ├── copy.h │ │ ├── copy_test.cc │ │ ├── dump_file.cc │ │ ├── dump_file.h │ │ ├── dump_file_test.cc │ │ ├── filter.cc │ │ ├── filter.h │ │ ├── filter_by_index.cc │ │ ├── filter_by_index.h │ │ ├── filter_by_index_test.cc │ │ ├── filter_test.cc │ │ ├── group.cc │ │ ├── group.h │ │ ├── group_agg.cc │ │ ├── group_agg.h │ │ ├── group_agg_test.cc │ │ ├── group_secret_agg.cc │ │ ├── group_secret_agg.h │ │ ├── group_secret_agg_test.cc │ │ ├── group_test.cc │ │ ├── if.cc │ │ ├── if.h │ │ ├── if_null.cc │ │ ├── if_null.h │ │ ├── if_null_test.cc │ │ ├── if_test.cc │ │ ├── in.cc │ │ ├── in.h │ │ ├── in_test.cc │ │ ├── insert_table.cc │ │ ├── insert_table.h │ │ ├── insert_table_mysql_pg_test.cc │ │ ├── insert_table_test.cc │ │ ├── is_null.cc │ │ ├── is_null.h │ │ ├── is_null_test.cc │ │ ├── join.cc │ │ ├── join.h │ │ ├── join_test.cc │ │ ├── limit.cc │ │ ├── limit.h │ │ ├── limit_test.cc │ │ ├── logical.cc │ │ ├── logical.h │ │ ├── logical_test.cc │ │ ├── make_private.cc │ │ ├── make_private.h │ │ ├── make_private_test.cc │ │ ├── make_public.cc │ │ ├── make_public.h │ │ ├── make_public_test.cc │ │ ├── make_share.cc │ │ ├── make_share.h │ │ ├── make_share_test.cc │ │ ├── oblivious_group_agg.cc │ │ ├── oblivious_group_agg.h │ │ ├── oblivious_group_agg_test.cc │ │ ├── oblivious_group_mark.cc │ │ ├── oblivious_group_mark.h │ │ ├── oblivious_group_mark_test.cc │ │ ├── publish.cc │ │ ├── publish.h │ │ ├── publish_test.cc │ │ ├── reduce.cc │ │ ├── reduce.h │ │ ├── reduce_test.cc │ │ ├── replicate.cc │ │ ├── replicate.h │ │ ├── replicate_test.cc │ │ ├── run_sql.cc │ │ ├── run_sql.h │ │ ├── run_sql_test.cc │ │ ├── secret_join.cc │ │ ├── secret_join.h │ │ ├── secret_join_test.cc │ │ ├── shape.cc │ │ ├── shape.h │ │ ├── shape_test.cc │ │ ├── shuffle.cc │ │ ├── shuffle.h │ │ ├── shuffle_test.cc │ │ ├── sort.cc │ │ ├── sort.h │ │ ├── sort_test.cc │ │ ├── test_util.cc │ │ ├── test_util.h │ │ ├── trigonometric.cc │ │ ├── trigonometric.h │ │ ├── trigonometric_test.cc │ │ ├── unary.cc │ │ ├── unary.h │ │ ├── unary_base.cc │ │ ├── unary_base.h │ │ ├── unary_test.cc │ │ ├── unique.cc │ │ ├── unique.h │ │ ├── unique_test.cc │ │ ├── window.cc │ │ ├── window.h │ │ └── window_test.cc │ ├── services/ │ │ ├── BUILD.bazel │ │ ├── engine_service_impl.cc │ │ ├── engine_service_impl.h │ │ ├── engine_service_impl_test.cc │ │ ├── error_collector_service.proto │ │ ├── error_collector_service_impl.cc │ │ ├── error_collector_service_impl.h │ │ ├── mock_report_service.proto │ │ ├── pipeline.cc │ │ ├── pipeline.h │ │ ├── prometheus_service.proto │ │ ├── prometheus_service_impl.cc │ │ ├── prometheus_service_impl.h │ │ ├── run_plan_core.cc │ │ └── run_plan_core.h │ └── util/ │ ├── BUILD.bazel │ ├── communicate_helper.h │ ├── concurrent_queue.h │ ├── context_util.cc │ ├── context_util.h │ ├── copy_to_proto_vistor.h │ ├── datamesh_helper.cc │ ├── datamesh_helper.h │ ├── datamesh_helper_test.cc │ ├── disk/ │ │ ├── BUILD.bazel │ │ ├── arrow_reader.cc │ │ ├── arrow_reader.h │ │ ├── arrow_writer.cc │ │ ├── arrow_writer.h │ │ ├── read_write_bench.cc │ │ └── reader_writer_test.cc │ ├── dp/ │ │ ├── BUILD.bazel │ │ └── flight.proto │ ├── filepath_helper.cc │ ├── filepath_helper.h │ ├── filepath_helper_test.cc │ ├── kpad_task_helper.cc │ ├── kpad_task_helper.h │ ├── kpad_task_helper_test.cc │ ├── logging.cc │ ├── logging.h │ ├── ndarray_to_arrow.cc │ ├── ndarray_to_arrow.h │ ├── prefix_sum.cc │ ├── prefix_sum.h │ ├── progress_util.h │ ├── prometheus_monitor.cc │ ├── prometheus_monitor.h │ ├── psi/ │ │ ├── BUILD.bazel │ │ ├── batch_provider.cc │ │ ├── batch_provider.h │ │ ├── batch_provider_test.cc │ │ ├── cipher_intersection.cc │ │ ├── cipher_intersection.h │ │ ├── cipher_intersection_bench.cc │ │ ├── cipher_intersection_test.cc │ │ ├── common.cc │ │ ├── common.h │ │ ├── detail_logger.cc │ │ ├── detail_logger.h │ │ ├── ub_helper.cc │ │ └── ub_helper.h │ ├── spu_io.cc │ ├── spu_io.h │ ├── ssl_helper.cc │ ├── ssl_helper.h │ ├── ssl_helper_test.cc │ ├── stringifier.cc │ ├── stringifier.h │ ├── stringifier_test.cc │ ├── table_util.cc │ ├── table_util.h │ ├── tensor_util.cc │ ├── tensor_util.h │ ├── time_util.cc │ ├── time_util.h │ ├── trace_categories.cc │ ├── trace_categories.h │ ├── upload_info_helper.cc │ ├── upload_info_helper.h │ └── upload_info_helper_test.cc ├── examples/ │ ├── opencore-demo/ │ │ └── main.go │ └── tutorial/ │ ├── .gitignore │ ├── README.md │ ├── docker/ │ │ ├── build.sh │ │ └── scql-ubuntu.Dockerfile │ ├── docker-compose.yml.template │ ├── engine/ │ │ ├── alice/ │ │ │ └── conf/ │ │ │ └── gflags.conf.template │ │ └── bob/ │ │ └── conf/ │ │ └── gflags.conf.template │ ├── example_config.json │ ├── mysql/ │ │ └── initdb/ │ │ ├── alice_init.sql │ │ └── bob_init.sql │ ├── project_bootstrap.sh │ └── setup.sh ├── go.mod ├── go.sum ├── pkg/ │ ├── config/ │ │ └── tls_config.go │ ├── constant/ │ │ └── constant.go │ ├── executor/ │ │ ├── engine_client.go │ │ ├── engine_client_test.go │ │ ├── engine_stub.go │ │ ├── engine_stub_mock.go │ │ ├── engine_stub_test.go │ │ ├── executor.go │ │ └── executor_test.go │ ├── expression/ │ │ ├── aggregation/ │ │ │ ├── aggregation.go │ │ │ ├── base_func.go │ │ │ ├── descriptor.go │ │ │ ├── util.go │ │ │ └── window_func.go │ │ ├── builtin.go │ │ ├── builtin_arithmetic.go │ │ ├── builtin_cast.go │ │ ├── builtin_compare.go │ │ ├── builtin_control.go │ │ ├── builtin_like.go │ │ ├── builtin_math.go │ │ ├── builtin_op.go │ │ ├── builtin_other.go │ │ ├── builtin_string.go │ │ ├── builtin_time.go │ │ ├── column.go │ │ ├── constant.go │ │ ├── errors.go │ │ ├── expression.go │ │ ├── expression_to_stmt.go │ │ ├── function_traits.go │ │ ├── helper.go │ │ ├── rand.go │ │ ├── scalar_function.go │ │ ├── schema.go │ │ ├── simple_rewriter.go │ │ └── util.go │ ├── infoschema/ │ │ ├── builder.go │ │ ├── infoschema.go │ │ ├── infoschema_test.go │ │ └── table_schema.go │ ├── interpreter/ │ │ ├── compiler/ │ │ │ ├── arrow_func_options.go │ │ │ ├── codegen_pass.go │ │ │ ├── column_security_relaxation.go │ │ │ ├── column_security_relaxation_test.go │ │ │ ├── common.go │ │ │ ├── common_test.go │ │ │ ├── compiler.go │ │ │ ├── compiler_test.go │ │ │ ├── constant.go │ │ │ ├── data/ │ │ │ │ └── test_queries.json │ │ │ ├── execution_graph_builder.go │ │ │ ├── execution_graph_builder_ops.go │ │ │ ├── execution_graph_builder_test.go │ │ │ ├── execution_graph_pass.go │ │ │ ├── inference.go │ │ │ ├── inference_test.go │ │ │ ├── kernel.go │ │ │ ├── kernel_resolver.go │ │ │ ├── kernel_resolver_test.go │ │ │ ├── kernel_test.go │ │ │ ├── logical_plan_pass.go │ │ │ ├── operator.go │ │ │ ├── operator_graph.go │ │ │ ├── operator_graph_builder.go │ │ │ ├── operator_graph_builder_test.go │ │ │ ├── operator_graph_pass.go │ │ │ ├── parser_pass.go │ │ │ ├── reverse_inference.go │ │ │ ├── reverse_inference_test.go │ │ │ ├── security_relaxation_manager.go │ │ │ ├── tensor_manager.go │ │ │ ├── tensor_manager_test.go │ │ │ ├── tensor_meta.go │ │ │ ├── tensor_meta_manager.go │ │ │ ├── tensor_meta_manager_test.go │ │ │ ├── tensor_meta_test.go │ │ │ ├── tensor_placement.go │ │ │ ├── tensor_placement_test.go │ │ │ ├── tensor_status_converter.go │ │ │ ├── tensor_status_converter_test.go │ │ │ ├── tensor_tracker.go │ │ │ ├── tensor_tracker_test.go │ │ │ ├── test_execution_graph_input.go │ │ │ ├── update_column_security.go │ │ │ ├── update_column_security_test.go │ │ │ ├── util/ │ │ │ │ ├── priority_queue.go │ │ │ │ └── priority_queue_test.go │ │ │ ├── visibility_analysis_pass.go │ │ │ ├── visibility_basic_inference.go │ │ │ ├── visibility_basic_inference_test.go │ │ │ ├── visibility_inference_with_relaxation.go │ │ │ ├── visibility_registry.go │ │ │ ├── visibility_registry_test.go │ │ │ ├── visibility_solver.go │ │ │ ├── visibility_solver_test.go │ │ │ ├── visible_parties.go │ │ │ └── visible_parties_test.go │ │ ├── graph/ │ │ │ ├── attribute.go │ │ │ ├── attribute_test.go │ │ │ ├── constant.go │ │ │ ├── data_type.go │ │ │ ├── engine_info.go │ │ │ ├── engine_info_test.go │ │ │ ├── execution_node.go │ │ │ ├── graph.go │ │ │ ├── graph_mapper.go │ │ │ ├── graph_mapper_test.go │ │ │ ├── graph_optimizer.go │ │ │ ├── graph_optimizer_test.go │ │ │ ├── graph_partitioner.go │ │ │ ├── graph_splitter.go │ │ │ ├── graph_splitter_test.go │ │ │ ├── graph_test.go │ │ │ └── tensor.go │ │ ├── operator/ │ │ │ ├── constant.go │ │ │ ├── operator_checker.go │ │ │ ├── operator_def.go │ │ │ ├── operator_def_test.go │ │ │ └── operator_registration.go │ │ └── sc/ │ │ └── scql_compiler.go │ ├── parser/ │ │ ├── .gitignore │ │ ├── LICENSE │ │ ├── Makefile │ │ ├── README.md │ │ ├── ast/ │ │ │ ├── advisor.go │ │ │ ├── ast.go │ │ │ ├── base.go │ │ │ ├── ddl.go │ │ │ ├── ddl_test.go │ │ │ ├── dml.go │ │ │ ├── dml_test.go │ │ │ ├── expressions.go │ │ │ ├── expressions_test.go │ │ │ ├── flag.go │ │ │ ├── flag_test.go │ │ │ ├── format_test.go │ │ │ ├── functions.go │ │ │ ├── functions_test.go │ │ │ ├── misc.go │ │ │ ├── misc_test.go │ │ │ ├── stats.go │ │ │ ├── util.go │ │ │ └── util_test.go │ │ ├── auth/ │ │ │ ├── auth.go │ │ │ └── auth_test.go │ │ ├── bench_test.go │ │ ├── charset/ │ │ │ ├── charset.go │ │ │ ├── charset_test.go │ │ │ └── encoding_table.go │ │ ├── consistent_test.go │ │ ├── digester.go │ │ ├── digester_test.go │ │ ├── export_test.go │ │ ├── format/ │ │ │ ├── format.go │ │ │ ├── format_dialect.go │ │ │ └── format_test.go │ │ ├── goyacc/ │ │ │ ├── format_yacc.go │ │ │ └── main.go │ │ ├── hintparser.go │ │ ├── hintparser.y │ │ ├── hintparser_test.go │ │ ├── hintparserimpl.go │ │ ├── lexer.go │ │ ├── lexer_test.go │ │ ├── misc.go │ │ ├── model/ │ │ │ ├── ddl.go │ │ │ ├── flags.go │ │ │ ├── model.go │ │ │ └── model_test.go │ │ ├── mysql/ │ │ │ ├── charset.go │ │ │ ├── const.go │ │ │ ├── const_test.go │ │ │ ├── errcode.go │ │ │ ├── errname.go │ │ │ ├── error.go │ │ │ ├── error_test.go │ │ │ ├── locale_format.go │ │ │ ├── state.go │ │ │ ├── type.go │ │ │ ├── type_test.go │ │ │ └── util.go │ │ ├── opcode/ │ │ │ ├── opcode.go │ │ │ └── opcode_test.go │ │ ├── parser.go │ │ ├── parser.y │ │ ├── parser_test.go │ │ ├── terror/ │ │ │ ├── terror.go │ │ │ └── terror_test.go │ │ ├── test.sh │ │ ├── test_driver/ │ │ │ ├── test_driver.go │ │ │ ├── test_driver_datum.go │ │ │ ├── test_driver_helper.go │ │ │ └── test_driver_mydecimal.go │ │ ├── types/ │ │ │ ├── etc.go │ │ │ ├── eval_type.go │ │ │ ├── field_type.go │ │ │ └── field_type_test.go │ │ └── yy_parser.go │ ├── planner/ │ │ ├── README.md │ │ ├── core/ │ │ │ ├── common_plans.go │ │ │ ├── database_dialect.go │ │ │ ├── dot.go │ │ │ ├── dot_test.go │ │ │ ├── errors.go │ │ │ ├── explain.go │ │ │ ├── expression_rewriter.go │ │ │ ├── hashcode.go │ │ │ ├── initialize.go │ │ │ ├── logical_plan_builder.go │ │ │ ├── logical_plan_test.go │ │ │ ├── logical_plans.go │ │ │ ├── logicalplan_to_stmt.go │ │ │ ├── logicalplan_to_stmt_helper.go │ │ │ ├── logicalplan_to_stmt_test.go │ │ │ ├── optimizer.go │ │ │ ├── plan.go │ │ │ ├── planbuilder.go │ │ │ ├── preprocess.go │ │ │ ├── rewrite.go │ │ │ ├── rewrite_test.go │ │ │ ├── rule_aggregation_elimination.go │ │ │ ├── rule_aggregation_push_down.go │ │ │ ├── rule_build_key_info.go │ │ │ ├── rule_column_pruning.go │ │ │ ├── rule_decorrelate.go │ │ │ ├── rule_eliminate_projection.go │ │ │ ├── rule_groupby_threshold.go │ │ │ ├── rule_join_reorder.go │ │ │ ├── rule_join_reorder_greedy.go │ │ │ ├── rule_merge_selection.go │ │ │ ├── rule_patch_timezone.go │ │ │ ├── rule_patch_timezone_test.go │ │ │ ├── rule_predicate_push_down.go │ │ │ ├── stringer.go │ │ │ ├── testdata/ │ │ │ │ ├── runsql_in.json │ │ │ │ ├── typical_query_in.json │ │ │ │ └── typical_query_out.json │ │ │ └── util.go │ │ ├── property/ │ │ │ └── property.go │ │ └── util/ │ │ ├── custom_vistor.go │ │ ├── custom_vistor_test.go │ │ ├── debug_util.go │ │ └── path.go │ ├── proto-gen/ │ │ ├── scql/ │ │ │ ├── common.pb.go │ │ │ ├── core.pb.go │ │ │ ├── engine.pb.go │ │ │ ├── interpreter.pb.go │ │ │ ├── scql_task.pb.go │ │ │ ├── status.pb.go │ │ │ ├── status_code.pb.go │ │ │ ├── subgraph.pb.go │ │ │ └── v1alpha1/ │ │ │ └── compiler.pb.go │ │ └── spu/ │ │ └── spu.pb.go │ ├── sessionctx/ │ │ ├── context.go │ │ ├── stmtctx/ │ │ │ └── stmtctx.go │ │ └── variable/ │ │ ├── session.go │ │ └── sysvar.go │ ├── status/ │ │ ├── status.go │ │ └── status_test.go │ ├── table/ │ │ ├── column.go │ │ ├── table.go │ │ └── tables/ │ │ └── tables.go │ ├── types/ │ │ ├── binary_literal.go │ │ ├── compare.go │ │ ├── convert.go │ │ ├── datum.go │ │ ├── errors.go │ │ ├── etc.go │ │ ├── eval_type.go │ │ ├── field_name.go │ │ ├── field_type.go │ │ ├── fsp.go │ │ ├── helper.go │ │ ├── mydecimal.go │ │ ├── mytime.go │ │ ├── overflow.go │ │ ├── parser_driver/ │ │ │ ├── value_expr.go │ │ │ └── value_expr_test.go │ │ └── time.go │ └── util/ │ ├── chunk/ │ │ ├── chunk.go │ │ ├── chunk_util.go │ │ ├── codec.go │ │ ├── column.go │ │ ├── compare.go │ │ ├── iterator.go │ │ └── row.go │ ├── codec/ │ │ ├── bytes.go │ │ ├── bytes_test.go │ │ ├── codec.go │ │ ├── codec_test.go │ │ ├── float.go │ │ └── number.go │ ├── execdetails/ │ │ └── execdetails.go │ ├── hack/ │ │ ├── hack.go │ │ └── hack_test.go │ ├── keyutil/ │ │ └── key_util.go │ ├── kusciaclient/ │ │ └── kusciaclient.go │ ├── logutil/ │ │ ├── log.go │ │ ├── monitor_log_entry.go │ │ └── monitor_log_entry_test.go │ ├── math/ │ │ ├── math.go │ │ └── math_test.go │ ├── mathutil/ │ │ ├── mathutil.go │ │ └── mathutil_wasm.go │ ├── message/ │ │ └── message_io_util.go │ ├── misc.go │ ├── mock/ │ │ ├── Makefile │ │ ├── README.md │ │ ├── mock.sh │ │ ├── mock_data.go │ │ ├── mock_db_data.py │ │ ├── mock_from_testdata.py │ │ ├── mock_schema.py │ │ └── testdata/ │ │ ├── db.json │ │ ├── generated_table_alice.json │ │ ├── generated_table_bob.json │ │ ├── generated_table_carol.json │ │ ├── table_alice.json │ │ ├── table_bob.json │ │ └── table_carol.json │ ├── mvmap/ │ │ ├── fnv.go │ │ ├── mvmap.go │ │ └── mvmap_test.go │ ├── parallel/ │ │ └── parallel.go │ ├── plancodec/ │ │ └── id.go │ ├── prometheus/ │ │ └── prom.go │ ├── ranger/ │ │ └── types.go │ ├── sliceutil/ │ │ ├── slice_util.go │ │ └── slice_util_test.go │ ├── sqlbuilder/ │ │ ├── sqlbuilder.go │ │ └── sqlbuilder_test.go │ ├── stringutil/ │ │ ├── string_util.go │ │ └── string_util_test.go │ ├── tableview/ │ │ └── tableview.go │ ├── testleak/ │ │ ├── fake.go │ │ └── leaktest.go │ ├── testutil/ │ │ └── testutil.go │ ├── texttree/ │ │ ├── texttree.go │ │ └── texttree_test.go │ ├── transaction/ │ │ └── tx.go │ └── url/ │ ├── url_util.go │ └── url_util_test.go ├── python/ │ ├── build.py │ ├── engine/ │ │ ├── BUILD.bazel │ │ └── engine_bindings.cc │ └── src/ │ ├── LICENSE │ ├── README.md │ ├── pyproject.toml │ ├── scql/ │ │ ├── __init__.py │ │ ├── compiler/ │ │ │ └── __init__.py │ │ └── engine/ │ │ └── __init__.py │ └── setup.py ├── renovate.json ├── scripts/ │ ├── check-whitespace.sh │ ├── format/ │ │ └── clang-format.sh │ └── graph-visualization/ │ ├── dot2js.py │ ├── graph.dot │ └── template.html ├── test-tools/ │ ├── README.md │ ├── ca_generator.sh │ └── find_uncover_err.py ├── version.txt └── version_build.sh ================================================ FILE CONTENTS ================================================ ================================================ FILE: .bazeliskrc ================================================ USE_BAZEL_VERSION=7.7.1 ================================================ FILE: .bazelrc ================================================ common --experimental_repo_remote_exec common --experimental_cc_shared_library common --experimental_ui_max_stdouterr_bytes=-1 common --experimental_remote_cache_compression common --experimental_remote_cache_compression_threshold=100 common --nolegacy_important_outputs common --remote_download_regex='.*\/scqlengine$' common --registry=https://raw.githubusercontent.com/secretflow/bazel-registry/main common --registry=https://bcr.bazel.build common --remote_download_outputs=all common --experimental_proto_descriptor_sets_include_source_info build --incompatible_new_actions_api=false build --copt=-fdiagnostics-color=always build --enable_platform_specific_config build --cxxopt=-std=c++17 build --host_cxxopt=-std=c++17 build --linkopt -fvisibility=hidden # default off CUDA build build --@rules_cuda//cuda:enable=false test --@rules_cuda//cuda:enable=false # Binary safety flags build --copt=-fPIC build --host_copt=-fstack-protector-strong build:linux --host_copt=-Wl,-z,noexecstack build:macos --host_copt=-Wa,--noexecstack test --keep_going test --test_output=errors test --test_timeout=1800 # platform specific config # Bazel will automatic pick platform config since we have enable_platform_specific_config set build:macos --copt=-Xclang=-fopenmp build:macos --copt=-Wno-unused-command-line-argument build:macos --features=-supports_dynamic_linker # build:macos --cxxopt -Wno-error=missing-template-arg-list-after-template-kw # build:macos --cxxopt -Wno-error=vla-cxx-extension build:macos --macos_minimum_os=13.0 build:macos --host_macos_minimum_os=13.0 build:macos --action_env MACOSX_DEPLOYMENT_TARGET=13.0 # static link libstdc++ & libgcc on Linux build:linux --copt=-fopenmp build:linux --linkopt=-fopenmp build:linux --action_env=BAZEL_LINKOPTS=-static-libstdc++:-static-libgcc build:linux --action_env=BAZEL_LINKLIBS=-l%:libstdc++.a:-l%:libgcc.a build:asan --strip=never build:asan --copt -fno-sanitize-recover=all build:asan --copt -fsanitize=address build:asan --copt -Og build:asan --copt -g build:asan --copt -fno-omit-frame-pointer build:asan --linkopt -fsanitize=address build:asan --define disable_tcmalloc=true build:asan --copt="-Wno-error=uninitialized" build:asan --copt="-Wno-error=maybe-uninitialized" build:asan --action_env=ASAN_OPTIONS=detect_odr_violation=0 ================================================ FILE: .circleci/config.yml ================================================ # Copyright 2023 Ant Group Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. version: 2.1 setup: true orbs: path-filtering: circleci/path-filtering@2.0.1 continuation: circleci/continuation@2.0.1 parameters: enable_regtest: type: boolean default: false workflows: run_ut: when: not: << pipeline.parameters.enable_regtest >> jobs: - path-filtering/filter: base-revision: main config-path: .circleci/lite-unittest-config.yml mapping: .circleci/path-filtering/unittest.conf run_regtest: when: << pipeline.parameters.enable_regtest >> jobs: - continuation/continue: configuration_path: .circleci/regtest-config.yml ================================================ FILE: .circleci/coverage-config.yml ================================================ # Copyright 2025 Ant Group Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Use the latest 2.1 version of CircleCI pipeline process engine. # See: https://circleci.com/docs/2.0/configuration-reference version: 2.1 orbs: coveralls: coveralls/coveralls@2.2.5 commands: setup_go_test: steps: - run: name: Install gcov2lcov command: | which gcov2lcov || go install github.com/jandelgado/gcov2lcov@latest setup_cpp_test: steps: - run: name: "Checkout devtools" command: git clone --depth=1 https://github.com/secretflow/devtools.git ../devtools - run: name: Setup BuildBuddy Cache command: ../devtools/bazel_cache_setup.py run_go_test: description: "Run go cov tests" steps: - run: name: "Go Cov Test" command: | set +e go mod tidy echo "Running tests with coverage..." go test -mod=readonly -timeout=30m -v -cover -race -coverprofile=coverage.tmp ./pkg/... cat coverage.tmp | grep -v '\.pb\.go:\|_mock\.go:' > coverage.out gcov2lcov -infile=coverage.out -outfile=coverage.lcov run_cpp_test: description: "Run cpp cov tests and collect artifacts" parameters: extra_bazel_args: type: string default: "" find_executable_flag: type: string default: "-executable" steps: - run: name: "Cpp Test" command: | set +e declare -i test_status echo "Running tests with coverage..." bazelisk --host_jvm_args=-Xmx8g coverage //engine/... \ << parameters.extra_bazel_args >> \ --combined_report=lcov \ --jobs=auto \ --ui_event_filters=-info,-debug,-warning \ --test_output=errors | tee test_result.log # Capture the exit status of the Bazel command test_status=${PIPESTATUS[0]} if [ ${test_status} -eq 0 ]; then echo "Processing coverage..." lcov --remove bazel-out/_coverage/_coverage_report.dat '*.pb.h' '*.pb.cc' -o bazel-out/_coverage/_coverage_report_filtered.dat else echo "Bazel coverage failed, skipping lcov processing. Archiving binaries and logs..." find bazel-bin/ << parameters.find_executable_flag >> -type f -name "*_test" -print0 | xargs -0 tar -cvzf test_binary.tar.gz find bazel-testlogs/ -type f -name "test.log" -print0 | xargs -0 tar -cvzf test_logs.tar.gz fi sh ../devtools/rename-junit-xml.sh exit ${test_status} # ref: https://support.circleci.com/hc/en-us/articles/14114124583195-How-to-set-a-custom-maximum-job-duration cancel_after_timeout: description: "Cancel job if it takes too long" parameters: timeout: type: string default: "60m" steps: - run: name: Set maximum job duration to << parameters.timeout >> background: true command: | sleep << parameters.timeout >> curl --request POST \ --url https://circleci.com/api/v2/project/gh/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/job/$CIRCLE_BUILD_NUM/cancel \ --header "Circle-Token: $CIRCLECI_API_TOKEN" jobs: linux_go_cov: docker: - image: secretflow/scql-ci:latest resource_class: "large" steps: - cancel_after_timeout: timeout: "30m" - checkout - setup_go_test - run_go_test - coveralls/upload: coverage_file: coverage.lcov coverage_format: lcov flag_name: "go-tests" linux_cpp_cov: docker: - image: secretflow/scql-ci:latest resource_class: "2xlarge" steps: - cancel_after_timeout: timeout: "90m" - checkout - run: name: "Install lcov" command: apt-get update && apt-get install -y lcov - setup_cpp_test - run_cpp_test: extra_bazel_args: "-c opt" find_executable_flag: "-executable" - coveralls/upload: coverage_file: bazel-out/_coverage/_coverage_report_filtered.dat coverage_format: lcov flag_name: "cpp-tests" - store_test_results: path: test-results - store_artifacts: when: on_fail path: test_binary.tar.gz - store_artifacts: when: on_fail path: test_logs.tar.gz workflows: run_cov: jobs: - linux_go_cov - linux_cpp_cov ================================================ FILE: .circleci/diff-cover-config.yml ================================================ # Copyright 2025 Ant Group Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Use the latest 2.1 version of CircleCI pipeline process engine. # See: https://circleci.com/docs/2.0/configuration-reference version: 2.1 parameters: base_commit: type: string description: "The base commit SHA for the diff." default: "origin/main" commands: setup_go_test: steps: - run: name: Install gcov2lcov command: | which gcov2lcov || go install github.com/jandelgado/gcov2lcov@latest setup_cpp_test: steps: - run: name: "Checkout devtools" command: git clone --depth=1 https://github.com/secretflow/devtools.git ../devtools - run: name: Setup BuildBuddy Cache command: ../devtools/bazel_cache_setup.py run_go_test: description: "Run go cov tests" steps: - run: name: "Go Cov Test" command: | set +e go mod tidy echo "Running tests with coverage..." go test -mod=readonly -timeout=30m -v -cover -race -coverprofile=coverage.tmp ./pkg/... cat coverage.tmp | grep -v '\.pb\.go:\|_mock\.go:' > coverage.out python ./test-tools/find_uncover_err.py coverage.out filtered_cover_no_err_branch.out gcov2lcov -infile=filtered_cover_no_err_branch.out -outfile=coverage.lcov run_cpp_test: description: "Run cpp cov tests and collect artifacts" parameters: extra_bazel_args: type: string default: "" find_executable_flag: type: string default: "-executable" steps: - run: name: "Cpp Test" command: | set +e declare -i test_status echo "Running tests with coverage..." bazelisk --host_jvm_args=-Xmx8g coverage //engine/... \ << parameters.extra_bazel_args >> \ --combined_report=lcov \ --jobs=auto \ --ui_event_filters=-info,-debug,-warning \ --test_output=errors | tee test_result.log # Capture the exit status of the Bazel command test_status=${PIPESTATUS[0]} if [ ${test_status} -eq 0 ]; then echo "Processing coverage..." lcov --remove bazel-out/_coverage/_coverage_report.dat '*.pb.h' '*.pb.cc' -o bazel-out/_coverage/_coverage_report_filtered.dat else echo "Bazel coverage failed, skipping lcov processing. Archiving binaries and logs..." find bazel-bin/ << parameters.find_executable_flag >> -type f -name "*_test" -print0 | xargs -0 tar -cvzf test_binary.tar.gz find bazel-testlogs/ -type f -name "test.log" -print0 | xargs -0 tar -cvzf test_logs.tar.gz fi sh ../devtools/rename-junit-xml.sh exit ${test_status} # ref: https://support.circleci.com/hc/en-us/articles/14114124583195-How-to-set-a-custom-maximum-job-duration cancel_after_timeout: description: "Cancel job if it takes too long" parameters: timeout: type: string default: "60m" steps: - run: name: Set maximum job duration to << parameters.timeout >> background: true command: | sleep << parameters.timeout >> curl --request POST \ --url https://circleci.com/api/v2/project/gh/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/job/$CIRCLE_BUILD_NUM/cancel \ --header "Circle-Token: $CIRCLECI_API_TOKEN" jobs: linux_diff_cov: docker: - image: secretflow/scql-ci:latest resource_class: "2xlarge" steps: - cancel_after_timeout: timeout: "90m" - checkout - run: name: "Install Dependencies (lcov, diff-cover)" command: | apt-get update && apt-get install -y lcov pip install diff-cover - setup_go_test - setup_cpp_test - run_go_test - run_cpp_test: extra_bazel_args: "-c opt" - run: name: "Debug Coverage Files" command: | echo "=== Debug Information ===" echo "Current working directory: $(pwd)" echo "Available coverage files:" find . -name "*.lcov" -o -name "*coverage*" -type f | head -10 if [ -f "coverage.lcov" ]; then echo "Go coverage file size: $(wc -l < coverage.lcov) lines" echo "Go coverage sample:" head -10 coverage.lcov fi if [ -f "bazel-out/_coverage/_coverage_report_filtered.dat" ]; then echo "C++ coverage file size: $(wc -l < bazel-out/_coverage/_coverage_report_filtered.dat) lines" echo "C++ coverage sample:" head -10 bazel-out/_coverage/_coverage_report_filtered.dat fi echo "Git status:" git status --porcelain echo "Recent commits:" git log --oneline -5 - run: name: "Calculate and Report Incremental Coverage" command: | echo "Calculating incremental coverage between << pipeline.parameters.base_commit >> and ${CIRCLE_SHA1}" git config remote.origin.url "https://github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}.git" git config advice.objectNameWarning false if ! git rev-parse --verify << pipeline.parameters.base_commit >> >/dev/null 2>&1; then echo "Base commit not found locally, fetching..." git fetch origin << pipeline.parameters.base_commit >> || \ git fetch origin --depth=100 || \ git fetch origin fi if ! git rev-parse --verify << pipeline.parameters.base_commit >> >/dev/null 2>&1; then echo "Error: Base commit << pipeline.parameters.base_commit >> not found" exit 1 fi if ! git rev-parse --verify ${CIRCLE_SHA1} >/dev/null 2>&1; then echo "Error: Current commit ${CIRCLE_SHA1} not found" exit 1 fi echo "Base commit: $(git rev-parse << pipeline.parameters.base_commit >>)" echo "Head commit: $(git rev-parse ${CIRCLE_SHA1})" # Create a directory to store the reports mkdir -p coverage-reports FINAL_EXIT_CODE=0 # --- 1. Go Incremental Coverage --- echo "" echo "--- Go Incremental Coverage ---" if [ -f "coverage.lcov" ]; then git diff << pipeline.parameters.base_commit >>..${CIRCLE_SHA1} -- '*.go' > go_diff.txt diff-cover coverage.lcov \ --diff-file=go_diff.txt \ --fail-under=80 \ --exclude='*_test.go' \ --exclude='*.pb.go' \ --exclude='*_mock.go' \ --format html:coverage-reports/go_diff_coverage.html \ || { echo "Go incremental coverage check FAILED."; FINAL_EXIT_CODE=1; } else echo "Go coverage file (coverage.lcov) not found. Skipping." fi # --- 2. C++ Incremental Coverage --- echo "" echo "--- C++ Incremental Coverage ---" CPP_COV_FILE="bazel-out/_coverage/_coverage_report_filtered.dat" if [ -f "${CPP_COV_FILE}" ]; then git diff << pipeline.parameters.base_commit >>..${CIRCLE_SHA1} -- '*.cpp' '*.h' '*.cc' > cpp_diff.txt diff-cover ${CPP_COV_FILE} \ --diff-file=cpp_diff.txt \ --fail-under=80 \ --exclude='*.pb.h' \ --exclude='*.pb.cc' \ --format html:coverage-reports/cpp_diff_coverage.html \ || { echo "C++ incremental coverage check FAILED."; FINAL_EXIT_CODE=1; } else echo "C++ coverage file (${CPP_COV_FILE}) not found. Skipping." fi exit ${FINAL_EXIT_CODE} - store_artifacts: path: coverage-reports - store_artifacts: path: coverage.lcov destination: go-coverage.lcov - store_artifacts: path: bazel-out/_coverage/_coverage_report_filtered.dat destination: cpp-coverage.lcov workflows: run_diff_cov_workflow: jobs: - linux_diff_cov ================================================ FILE: .circleci/full-unittest-config.yml ================================================ # Copyright 2023 Ant Group Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Use the latest 2.1 version of CircleCI pipeline process engine. # See: https://circleci.com/docs/2.0/configuration-reference version: 2.1 commands: setup_cpp_test: steps: - run: name: "Checkout devtools" command: git clone --depth=1 https://github.com/secretflow/devtools.git ../devtools - run: name: Setup BuildBuddy Cache command: ../devtools/bazel_cache_setup.py run_go_test: description: "Run go tests" steps: - run: name: "Go Test" command: | set +e go mod tidy echo "Running unit tests..." go test -mod=readonly -timeout=30m -v -short ./pkg/... run_cpp_test: description: "Run cpp tests and collect artifacts on fail" parameters: extra_bazel_args: type: string default: "" find_executable_flag: type: string default: "-executable" steps: - run: name: "Cpp Test" command: | set +e declare -i test_status echo "Running unit tests..." bazelisk --host_jvm_args=-Xmx8g test //engine/... \ << parameters.extra_bazel_args >> \ --jobs=auto \ --ui_event_filters=-info,-debug,-warning \ --test_output=errors | tee test_result.log # Capture the exit status of the Bazel command test_status=${PIPESTATUS[0]} sh ../devtools/rename-junit-xml.sh if [ ${test_status} -ne 0 ]; then echo "Tests failed. Archiving binaries and logs..." find bazel-bin/ << parameters.find_executable_flag >> -type f -name "*_test" -print0 | xargs -0 tar -cvzf test_binary.tar.gz find bazel-testlogs/ -type f -name "test.log" -print0 | xargs -0 tar -cvzf test_logs.tar.gz fi exit ${test_status} # ref: https://support.circleci.com/hc/en-us/articles/14114124583195-How-to-set-a-custom-maximum-job-duration cancel_after_timeout: description: "Cancel job if it takes too long" parameters: timeout: type: string default: "60m" steps: - run: name: Set maximum job duration to << parameters.timeout >> background: true command: | sleep << parameters.timeout >> curl --request POST \ --url https://circleci.com/api/v2/project/gh/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/job/$CIRCLE_BUILD_NUM/cancel \ --header "Circle-Token: $CIRCLECI_API_TOKEN" jobs: linux_go_ut: docker: - image: secretflow/scql-ci:latest parameters: resource_class: type: string resource_class: << parameters.resource_class >> steps: - cancel_after_timeout: timeout: "20m" - checkout - run_go_test linux_cpp_ut: docker: - image: secretflow/scql-ci:latest parameters: resource_class: type: string resource_class: << parameters.resource_class >> steps: - cancel_after_timeout: timeout: "60m" - checkout - setup_cpp_test - run_cpp_test: extra_bazel_args: "-c opt" find_executable_flag: "-executable" - store_test_results: path: test-results - store_artifacts: when: on_fail path: test_binary.tar.gz - store_artifacts: when: on_fail path: test_logs.tar.gz macOS_go_ut: macos: xcode: 16.3.0 resource_class: m4pro.medium steps: - cancel_after_timeout: timeout: "20m" - checkout - run: name: "Install homebrew dependencies" command: | brew install wget go - run_go_test macOS_cpp_ut: macos: xcode: 16.3.0 resource_class: m4pro.medium steps: - cancel_after_timeout: timeout: "60m" - checkout - run: name: "Install homebrew dependencies" command: | brew install bazelisk cmake ninja libomp wget go@1.24 md5sha1sum brew link go@1.24 - setup_cpp_test - run_cpp_test: extra_bazel_args: "" find_executable_flag: "-perm +111" - store_test_results: path: test-results - store_artifacts: when: on_fail path: test_binary.tar.gz - store_artifacts: when: on_fail path: test_logs.tar.gz workflows: run_go_ut: jobs: - linux_go_ut: matrix: parameters: resource_class: ["large", "arm.large"] - macOS_go_ut run_cpp_ut: jobs: - linux_cpp_ut: matrix: parameters: resource_class: ["2xlarge", "arm.2xlarge"] - macOS_cpp_ut ================================================ FILE: .circleci/lite-unittest-config.yml ================================================ # Copyright 2025 Ant Group Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Use the latest 2.1 version of CircleCI pipeline process engine. # See: https://circleci.com/docs/2.0/configuration-reference version: 2.1 parameters: run_go_ut: type: boolean default: false run_cpp_ut: type: boolean default: false commands: setup_cpp_test: steps: - run: name: "Checkout devtools" command: git clone --depth=1 https://github.com/secretflow/devtools.git ../devtools - run: name: Setup BuildBuddy Cache command: ../devtools/bazel_cache_setup.py run_go_test: description: "Run go tests" steps: - run: name: "Go Test" command: | set +e go mod tidy echo "Running unit tests..." go test -mod=readonly -timeout=30m -v -short ./pkg/... run_cpp_test: description: "Run cpp tests and collect artifacts on fail" steps: - run: name: "Cpp Test" command: | set +e declare -i test_status echo "Running unit tests..." bazelisk --host_jvm_args=-Xmx8g test //engine/... \ -c opt \ --jobs=16 \ --local_ram_resources=20480 \ --ui_event_filters=-info,-debug,-warning \ --test_output=errors | tee test_result.log # Capture the exit status of the Bazel command test_status=${PIPESTATUS[0]} sh ../devtools/rename-junit-xml.sh if [ ${test_status} -ne 0 ]; then echo "Tests failed. Archiving binaries and logs..." find bazel-bin/ -executable -type f -name "*_test" -print0 | xargs -0 tar -cvzf test_binary.tar.gz find bazel-testlogs/ -type f -name "test.log" -print0 | xargs -0 tar -cvzf test_logs.tar.gz fi exit ${test_status} # ref: https://support.circleci.com/hc/en-us/articles/14114124583195-How-to-set-a-custom-maximum-job-duration cancel_after_timeout: description: "Cancel job if it takes too long" parameters: timeout: type: string default: "60m" steps: - run: name: Set maximum job duration to << parameters.timeout >> background: true command: | sleep << parameters.timeout >> curl --request POST \ --url https://circleci.com/api/v2/project/gh/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/job/$CIRCLE_BUILD_NUM/cancel \ --header "Circle-Token: $CIRCLECI_API_TOKEN" jobs: linux_go_ut: docker: - image: secretflow/scql-ci:latest resource_class: "large" steps: - cancel_after_timeout: timeout: "20m" - checkout - run_go_test linux_cpp_ut: docker: - image: secretflow/scql-ci:latest resource_class: "2xlarge" steps: - cancel_after_timeout: timeout: "60m" - checkout - setup_cpp_test - run_cpp_test - store_test_results: path: test-results - store_artifacts: when: on_fail path: test_binary.tar.gz - store_artifacts: when: on_fail path: test_logs.tar.gz workflows: run_go_ut: when: << pipeline.parameters.run_go_ut >> jobs: - linux_go_ut run_cpp_ut: when: << pipeline.parameters.run_cpp_ut >> jobs: - linux_cpp_ut ================================================ FILE: .circleci/path-filtering/unittest.conf ================================================ api/.* run_go_ut true cmd/.* run_go_ut true pkg/.* run_go_ut true .circleci/lite-unittest-config.yml run_go_ut true bazel/.* run_cpp_ut true engine/.* run_cpp_ut true .bazelrc run_cpp_ut true .bazeliskrc run_cpp_ut true MODULE.bazel run_cpp_ut true .circleci/lite-unittest-config.yml run_cpp_ut true ================================================ FILE: .clang-format ================================================ # Use the Google style in this project. BasedOnStyle: Google IncludeBlocks: Regroup IncludeCategories: - Regex: '^<.*\.h>' Priority: 1 - Regex: "^<.*" Priority: 2 - Regex: '.*\.pb\.h"$' Priority: 5 - Regex: '^"engine.*' Priority: 4 - Regex: '^".*' Priority: 3 ================================================ FILE: .clang-tidy ================================================ Checks: "abseil-cleanup-ctad, abseil-faster-strsplit-delimiter, abseil-duration-*, abseil-no-namespace, abseil-redundant-strcat-calls, abseil-str-cat-append, abseil-string-find-startswith, abseil-upgrade-duration-conversions bugprone-*, -bugprone-easily-swappable-parameters, -bugprone-implicit-widening-of-multiplication-result, -bugprone-narrowing-conversions, google-build-using-namespace, google-explicit-constructor, google-global-names-in-headers, google-readability-casting, google-runtime-int, google-runtime-operator, misc-unused-using-decls, modernize-*, -modernize-use-trailing-return-type, -modernize-avoid-c-arrays, -modernize-return-braced-init-list, -modernize-use-nodiscard, performance-*, readability-*, -readability-else-after-return, -readability-identifier-length, -readability-function-cognitive-complexity, -readability-magic-numbers, -readability-named-parameter, -readability-math-missing-parentheses, -readability-redundant-access-specifiers, -readability-simplify-boolean-expr, concurrency-mt-unsafe" CheckOptions: - key: bugprone-argument-comment.StrictMode value: 1 - key: bugprone-dangling-handle.HandleClasses value: "std::basic_string_view;std::experimental::basic_string_view;absl::string_view" - key: misc-non-private-member-variables-in-classes.IgnoreClassesWithAllMemberVariablesBeingPublic value: 1 # Ignore GoogleTest function macros. - key: readability-identifier-naming.FunctionIgnoredRegexp value: "(TEST|TEST_F|TEST_P|INSTANTIATE_TEST_SUITE_P|MOCK_METHOD|TYPED_TEST)" - key: readability-identifier-naming.ClassCase value: "CamelCase" - key: readability-identifier-naming.EnumCase value: "CamelCase" - key: readability-identifier-naming.EnumConstantCase value: "CamelCase" - key: readability-identifier-naming.ParameterCase value: "lower_case" - key: readability-identifier-naming.PrivateMemberCase value: "lower_case" - key: readability-identifier-naming.PrivateMemberSuffix value: "_" - key: readability-identifier-naming.StructCase value: "CamelCase" - key: readability-identifier-naming.TypeAliasCase value: "CamelCase" - key: readability-identifier-naming.UnionCase value: "CamelCase" - key: readability-identifier-naming.FunctionCase value: "CamelBack" ================================================ FILE: .coveralls.yml ================================================ # Configure whether the Coveralls status check fails when coverage decreases fail_on_coverage_decrease: false coverage: status: project: go-tests: # Set a coverage threshold for the changed lines of code. target: 70% cpp-tests: target: 70% patch: default: target: 70% comment: pull_request: # update a comment behavior: default # diff: Shows overall coverage change and coverage of changed lines # flags: If upload multiple reports (C++ and Go), they will be displayed separately # files: Lists the files where coverage has changed layout: "diff, flags, files" ================================================ FILE: .devcontainer/Dockerfile ================================================ FROM secretflow/ubuntu-base-ci:latest ARG TARGETPLATFORM ARG GO_VERSION=1.24.0 # install go RUN if [ "$TARGETPLATFORM" = "linux/arm64" ] ; \ then \ GO_ARCH=arm64 && \ GO_SHA256SUM=c3fa6d16ffa261091a5617145553c71d21435ce547e44cc6dfb7470865527cc7 ; \ else \ GO_ARCH=amd64 && \ GO_SHA256SUM=dea9ca38a0b852a74e81c26134671af7c0fbe65d81b0dc1c5bfe22cf7d4c8858 ; \ fi \ && url="https://golang.google.cn/dl/go${GO_VERSION}.linux-${GO_ARCH}.tar.gz"; \ wget --no-check-certificate -O go.tgz "$url"; \ echo "${GO_SHA256SUM} *go.tgz" | sha256sum -c -; \ tar -C /usr/local -xzf go.tgz; \ rm go.tgz; ENV GOPATH="/usr/local" ENV PATH="/usr/local/go/bin:${GOPATH}/bin:${PATH}" RUN apt update \ && apt upgrade -y \ && apt install -y protobuf-compiler \ && apt clean # Create a non-root user 'vscode' with sudo privileges # The user ID 1000 is common for the primary user in Linux distributions. # The GID 1000 corresponds to the user's primary group. RUN groupadd --gid 1000 vscode && \ useradd --uid 1000 --gid 1000 --shell /bin/bash --create-home vscode && \ echo "vscode ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers # Set the default user for subsequent commands USER vscode # Set the working directory inside the container WORKDIR /workspaces/scql # Set default shell to bash for the vscode user ENV SHELL /bin/bash # You can add more SCQL-specific dependencies here if needed, # although 'make install-dev-deps' in postCreateCommand should handle most project dependencies. # Keep the container running (optional, useful for debugging setup) # CMD ["sleep", "infinity"] ================================================ FILE: .devcontainer/devcontainer.json ================================================ // For format details, see https://aka.ms/devcontainer.json. For config options, see the // README at: https://github.com/devcontainers/templates/tree/main/src/cpp { // A name for the dev container which can be displayed in UI. "name": "SCQL Dev Container", // Sets the run context to one level up instead of the .devcontainer folder. "context": "..", // Update the VARIANT arg in docker-compose.yml to pick a Debian OS version: bullseye, buster // Use Python 3.10 to align with SCQL's CI/CD environment if possible, adjust if needed. "build": { "dockerfile": "Dockerfile" }, // Features to add to the dev container. More info: https://containers.dev/features. // "features": {}, // Use 'forwardPorts' to make a list of ports inside the container available locally. // "forwardPorts": [], // Use 'postCreateCommand' to run commands after the container is created. // Installs development dependencies using the Makefile target. "postCreateCommand": "echo 'hello world'", // Configure tool-specific properties. // "customizations": {}, // Specifies the user the container will run as. Default is root. // Using a non-root user 'vscode' is recommended for security. "remoteUser": "vscode", // Mount the workspace folder. "workspaceFolder": "/workspaces/scql", "workspaceMount": "source=${localWorkspaceFolder},target=/workspaces/scql,type=bind,consistency=cached" } ================================================ FILE: .github/CODEOWNERS ================================================ # default reviewers for everything in this repo. * @secretflow/scql-dev ================================================ FILE: .github/ISSUE_TEMPLATE/Consulting_issue_template.yaml ================================================ name: Consulting Template description: Ask SCQL related questions body: - type: markdown attributes: value: | Please ensure that you are reporting the consultation issue on GitHub.（CCL、Perfomance、Feature、Documentation、Others） - type: dropdown id: issue-type attributes: label: Issue Type description: What type of issue would you like to report? multiple: false options: - CCL - Perfomance - Feature - Documentation - Others validations: required: true - type: dropdown id: searched-for-existing-issues attributes: label: Have you searched for existing issues? description: It is recommended to search existing [documentation](https://www.secretflow.org.cn/zh-CN/docs/scql/main/topics) and [issues](https://github.com/secretflow/scql/issues) first options: - 'Yes' - 'No' validations: required: true - type: input id: link attributes: label: Link to Relevant Documentation description: For faster problem-solving, if there are relevant documents, please attach links. placeholder: e.g., https://www.secretflow.org.cn/zh-CN/docs/scql/main/topics/ccl/intro validations: required: false - type: textarea id: Question-Details attributes: label: Question Details description: Please detail your issue with observed versus expected behavior and attempted solutions to expedite resolution. placeholder: Describe the questions you want to consult and what you want to do value: render: shell validations: required: true ================================================ FILE: .github/ISSUE_TEMPLATE/Error_Issue_Template.yaml ================================================ name: Error Template description: Thank you for reporting the issue! body: - type: markdown attributes: value: | Please ensure that you are reporting the consultation issue on GitHub.（Install/Build、Running、CCL、Others） - type: dropdown id: issue-type attributes: label: Issue Type description: What type of issue would you like to report? multiple: false options: - Install/Build - Running - CCL - Others validations: required: true - type: dropdown id: searched-for-existing-issues attributes: label: Have you searched for existing issues? description: It is recommended to search existing [documentation](https://www.secretflow.org.cn/zh-CN/docs/scql/main/topics) and [issues](https://github.com/secretflow/scql/issues) first options: - 'Yes' - 'No' validations: required: true - type: input id: OS attributes: label: OS Platform and Distribution description: placeholder: e.g., Linux Ubuntu 18.04 validations: required: true - type: input id: scql-version attributes: label: SCQL Version description: placeholder: e.g., SCQL 0.7.0b0 validations: required: true - type: textarea id: what-happened attributes: label: What happend and What you expected to happen. description: A clear and concise description of what the bug is. placeholder: Describe the bug, expected behavior. value: render: shell validations: required: true - type: textarea id: scql-config attributes: label: Configuration used to run SCQL. description: Supply SCQL runtime config (.yaml, .conf) and, for CCL issues, supply authorization info and SQL statements. placeholder: | # - For install/deploy, provide files ending with `.yaml` and `.conf`. # - For CCL, provide the authorization details and CCL SQL. value: render: shell validations: required: true - type: textarea id: log-output attributes: label: SCQL log output. description: Supply relevant log output (docker logs -f xxx-broker-xxx/xxx-engine-xxx); For multi-party (e.g. Alice, Bob) scenarios, include all corresponding logs. placeholder: | # alice.log ....... # bob.log ....... value: render: shell validations: required: true ================================================ FILE: .github/workflows/black.yml ================================================ --- name: Python Linter on: push: branches: - main paths: - '**.py' pull_request: branches: - main paths: - '**.py' permissions: contents: read jobs: python-linter: uses: secretflow/.github/.github/workflows/python-linter.yml@main ================================================ FILE: .github/workflows/buildifier.yml ================================================ --- name: Bazel files linter on: push: branches: - main pull_request: branches: - main permissions: contents: read jobs: bazel-formatting-check: uses: secretflow/.github/.github/workflows/bazel-linter.yml@main ================================================ FILE: .github/workflows/cla.yml ================================================ --- name: CLA Assistant on: issue_comment: types: [created] pull_request_target: types: [opened, closed, synchronize] jobs: CLAssistant: uses: secretflow/.github/.github/workflows/cla.yml@main secrets: inherit ================================================ FILE: .github/workflows/clang-format-linter.yml ================================================ --- name: Run clang-format Linter on: push: branches: - main paths: - '**.cc' - '**.cpp' - '**.hpp' - '**.h' - '**.proto' pull_request: branches: - main paths: - '**.cc' - '**.cpp' - '**.hpp' - '**.h' - '**.proto' permissions: contents: read jobs: run-clang-format: uses: secretflow/.github/.github/workflows/clang-format.yml@main ================================================ FILE: .github/workflows/codeql.yml ================================================ # For most projects, this workflow file will not need changing; you simply need # to commit it to your repository. # # You may wish to alter this file to override the set of languages analyzed, # or to provide custom queries or build logic. # # ******** NOTE ******** # We have attempted to detect the languages in your repository. Please check # the `language` matrix defined below to confirm you have the correct set of # supported CodeQL languages. # name: "CodeQL" on: push: branches: ["main"] pull_request: # The branches below must be a subset of the branches above branches: ["main"] schedule: - cron: "0 0 * * 1" permissions: contents: read jobs: analyze: name: Analyze runs-on: ubuntu-latest permissions: actions: read contents: read security-events: write strategy: fail-fast: false matrix: language: ["cpp", "go", "python"] # CodeQL supports [ $supported-codeql-languages ] # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support steps: - name: Harden the runner (Audit all outbound calls) uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0 with: egress-policy: audit - name: Checkout repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@28deaeda66b76a05916b6923827895f2b14ab387 # v3.28.16 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild uses: github/codeql-action/autobuild@28deaeda66b76a05916b6923827895f2b14ab387 # v3.28.16 # ℹ️ Command-line programs to run using the OS shell. # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun # If the Autobuild fails above, remove it and uncomment the following three lines. # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. # - run: | # echo "Run, Build Application using script" # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@28deaeda66b76a05916b6923827895f2b14ab387 # v3.28.16 with: category: "/language:${{matrix.language}}" ================================================ FILE: .github/workflows/dependency-review.yml ================================================ # Dependency Review Action # # This Action will scan dependency manifest files that change as part of a Pull Request, # surfacing known-vulnerable versions of the packages declared or updated in the PR. # Once installed, if the workflow run is marked as required, # PRs introducing known-vulnerable packages will be blocked from merging. # # Source repository: https://github.com/actions/dependency-review-action name: 'Dependency Review' on: [pull_request] permissions: contents: read jobs: dependency-review: runs-on: ubuntu-latest steps: - name: Harden the runner (Audit all outbound calls) uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0 with: egress-policy: audit - name: 'Checkout Repository' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: 'Dependency Review' uses: actions/dependency-review-action@67d4f4bd7a9b17a0db54d2a7519187c65e339de8 # v4 ================================================ FILE: .github/workflows/docs-check.yml ================================================ --- name: Check Docs on: push: branches: - main paths: - 'docs/**' pull_request: branches: - main paths: - 'docs/**' permissions: contents: read pull-requests: read jobs: check-docs: name: check docs runs-on: [ubuntu-latest] container: image: secretflow/scql-ci:20250228 steps: - name: Harden the runner (Audit all outbound calls) uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0 with: egress-policy: audit - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: '0' - name: make docs run: | # avoid {{ and {# in docs if [ $(grep -E '({{|{#)' --include=\*.{rst,md,ipynb,po} --exclude=run-scql-on-kuscia.* -rnw 'docs' | wc -l) != 0 ]; then echo "({{|{#) is not allowed in rst,md,ipynb,po files. Thank you for cooperation." grep -E '({{|{#)' --include=\*.{rst,md,ipynb,po} --exclude=run-scql-on-kuscia.* -rnw 'docs' exit 1 fi ================================================ FILE: .github/workflows/docs-publish.yml ================================================ # Copyright 2025 Ant Group Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. name: Publish Docs on: push: branches: - main tags: - "*" jobs: check-docs: name: check docs runs-on: [ubuntu-latest] steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: node-version: "22" - uses: actions/setup-python@v5 with: python-version: "3.10" - name: pubilsh docs run: | python3 -m venv ~/.venv/docs source ~/.venv/docs/bin/activate python -m pip install -r docs/requirements.txt secretflow-doctools build --lang zh_CN --lang en secretflow-doctools publish \ --name @secretflow/x-scql \ --index-js docs/_build/esm/index.js env: DRY_RUN: "0" # omit in test runs DOCTOOLS_PUBLISH_NPM_TOKEN: ${{secrets.DOCTOOLS_PUBLISH_NPM_TOKEN}} ================================================ FILE: .github/workflows/golangci-lint.yml ================================================ --- name: Golangci-Lint on: push: branches: - main paths: - '**.go' - '.golangci-lint.yml' pull_request: branches: - main paths: - '**.go' - '.golangci-lint.yml' env: GO_VERSION: 1.24 GOLANGCI_LINT_VERSION: v2.0 permissions: # Required: allow read access to the content for analysis. contents: read # Optional: allow read access to pull request. Use with `only-new-issues` option. pull-requests: read jobs: detect-modules: runs-on: ubuntu-latest outputs: modules: ${{ steps.set-modules.outputs.modules }} steps: - name: Harden the runner (Audit all outbound calls) uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0 with: egress-policy: audit - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0 with: go-version: ${{ env.GO_VERSION }} - id: set-modules run: echo "modules=$(go list -m -json | jq -s '.' | jq -c '[.[].Dir]')" >> $GITHUB_OUTPUT golangci-lint: needs: detect-modules runs-on: ubuntu-latest strategy: matrix: modules: ${{ fromJSON(needs.detect-modules.outputs.modules) }} steps: - name: Harden the runner (Audit all outbound calls) uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0 with: egress-policy: audit - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0 with: go-version: ${{ env.GO_VERSION }} - name: golangci-lint ${{ matrix.modules }} uses: golangci/golangci-lint-action@1481404843c368bc19ca9406f87d6e0fc97bdcfd # v7.0.0 with: version: ${{ env.GOLANGCI_LINT_VERSION }} working-directory: ${{ matrix.modules }} ================================================ FILE: .github/workflows/govulncheck.yml ================================================ --- name: Go Vulnerability Check on: push: branches: - main paths: - '**.go' - 'go.mod' - 'go.sum' - '.github/workflows/govulncheck.yml' pull_request: branches: - main paths: - '**.go' - 'go.mod' - 'go.sum' - '.github/workflows/govulncheck.yml' jobs: govulncheck_job: runs-on: ubuntu-latest name: Run govulncheck steps: - id: govulncheck uses: golang/govulncheck-action@v1 with: go-package: ./... go-version-file: 'go.mod' output-format: 'text' ================================================ FILE: .github/workflows/license-check.yml ================================================ --- name: License Check on: push: branches: - main pull_request: branches: - main permissions: contents: read jobs: license-checker: uses: secretflow/.github/.github/workflows/license-check.yml@main ================================================ FILE: .github/workflows/oscp.yml ================================================ name: Unassign Stale OSCP Issues on: schedule: - cron: "0 */6 * * *" # Every 6 hours jobs: unassign-stale-issues: uses: secretflow/.github/.github/workflows/oscp-unassign.yml@main ================================================ FILE: .github/workflows/scorecards.yml ================================================ # This workflow uses actions that are not certified by GitHub. They are provided # by a third-party and are governed by separate terms of service, privacy # policy, and support documentation. name: Scorecard supply-chain security on: # For Branch-Protection check. Only the default branch is supported. See # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection branch_protection_rule: # To guarantee Maintained check is occasionally updated. See # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained schedule: - cron: '20 7 * * 2' push: branches: ["main"] # Declare default permissions as read only. permissions: read-all jobs: analysis: name: Scorecard analysis runs-on: ubuntu-latest permissions: # Needed to upload the results to code-scanning dashboard. security-events: write # Needed to publish results and get a badge (see publish_results below). id-token: write contents: read actions: read # To allow GraphQL ListCommits to work issues: read pull-requests: read # To detect SAST tools checks: read steps: - name: Harden the runner (Audit all outbound calls) uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0 with: egress-policy: audit - name: "Checkout code" uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - name: "Run analysis" uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # v2.4.0 with: results_file: results.sarif results_format: sarif # (Optional) "write" PAT token. Uncomment the `repo_token` line below if: # - you want to enable the Branch-Protection check on a *public* repository, or # - you are installing Scorecards on a *private* repository # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat. # repo_token: ${{ secrets.SCORECARD_TOKEN }} # Public repositories: # - Publish results to OpenSSF REST API for easy access by consumers # - Allows the repository to include the Scorecard badge. # - See https://github.com/ossf/scorecard-action#publishing-results. # For private repositories: # - `publish_results` will always be set to `false`, regardless # of the value entered here. publish_results: true # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: SARIF file path: results.sarif retention-days: 5 # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" uses: github/codeql-action/upload-sarif@28deaeda66b76a05916b6923827895f2b14ab387 # v3.28.16 with: sarif_file: results.sarif ================================================ FILE: .github/workflows/stale.yml ================================================ --- name: Mark stale issues and pull requests on: workflow_dispatch: schedule: - cron: 40 9 * * * jobs: stale: uses: secretflow/.github/.github/workflows/stale.yml@main ================================================ FILE: .github/workflows/trigger-ci-cov.yml ================================================ name: Trigger CircleCI Coverage Test on: pull_request_target: types: [labeled] # Only run when a label is added jobs: trigger-circleci: runs-on: ubuntu-latest # This job only runs if the label that was just added is exactly "run-ci-cov". if: github.event.label.name == 'run-ci-cov' permissions: pull-requests: write steps: - name: Trigger CircleCI Pipeline run: | echo "Label 'run-ci-cov' was added to PR #${{ github.event.pull_request.number }}. Triggering CircleCI pipeline..." # We use curl to directly call the CircleCI API v2. # This is more flexible than the official trigger action. curl -X POST \ --url "https://circleci.com/api/v2/project/github/secretflow/scql/pipeline/run" \ --header "Content-Type: application/json" \ --header "Circle-Token: ${{ secrets.CCI_TOKEN }}" \ --data '{"definition_id":"425b19a9-9f21-4d53-a4a6-c2e57939ddd0","config":{"branch":"pull/${{ github.event.pull_request.number }}/head"},"checkout":{"branch":"pull/${{ github.event.pull_request.number }}/head"}}' - name: Remove Label from PR run: | echo "Removing 'run-ci-cov' label from PR #${{ github.event.pull_request.number }}..." curl -s -X DELETE \ -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ -H "Accept: application/vnd.github.v3+json" \ "https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/labels/run-ci-cov" ================================================ FILE: .github/workflows/trigger-ci-full-ut.yml ================================================ name: Trigger CircleCI Full Unit Test for Forked & Labeled PR on: pull_request_target: types: [labeled] jobs: trigger-circleci: runs-on: ubuntu-latest # This job only runs if the label that was just added is exactly "run-ci-full". if: github.event.label.name == 'run-ci-full' permissions: pull-requests: write steps: - name: Trigger CircleCI Pipeline run: | echo "Label 'run-ci-full' was added to PR #${{ github.event.pull_request.number }}. Triggering CircleCI pipeline..." curl -X POST \ --url "https://circleci.com/api/v2/project/github/secretflow/scql/pipeline/run" \ --header "Content-Type: application/json" \ --header "Circle-Token: ${{ secrets.CCI_TOKEN }}" \ --data '{"definition_id":"ee28d7c2-ef66-4a57-bc90-6895e2b55f14","config":{"branch":"pull/${{ github.event.pull_request.number }}/head"},"checkout":{"branch":"pull/${{ github.event.pull_request.number }}/head"}}' - name: Remove Label from PR run: | echo "Removing 'run-ci-full' label from PR #${{ github.event.pull_request.number }}..." curl -s -X DELETE \ -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ -H "Accept: application/vnd.github.v3+json" \ "https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/labels/run-ci-full" ================================================ FILE: .github/workflows/trigger-ci-lite-ut.yml ================================================ name: Trigger CircleCI Lite Unit Test for Forked & Labeled PR on: pull_request_target: types: [labeled] jobs: trigger-circleci: runs-on: ubuntu-latest # This job only runs under two conditions: # 1. The PR is from a fork. # 2. The label that was just added is exactly "run-ci". if: | github.event.pull_request.head.repo.full_name != github.repository && (github.event.action == 'labeled' && github.event.label.name == 'run-ci') permissions: pull-requests: write steps: - name: Trigger CircleCI Pipeline run: | echo "Forked PR #${{ github.event.pull_request.number }} has 'run-ci' label. Triggering CircleCI pipeline..." curl -X POST \ --url "https://circleci.com/api/v2/project/github/secretflow/scql/pipeline/run" \ --header "Content-Type: application/json" \ --header "Circle-Token: ${{ secrets.CCI_TOKEN }}" \ --data '{"definition_id":"c6f435ca-697f-488c-83da-156862bb7392","config":{"branch":"pull/${{ github.event.pull_request.number }}/head"},"checkout":{"branch":"pull/${{ github.event.pull_request.number }}/head"}}' - name: Remove Label from PR run: | echo "Removing 'run-ci' label from PR #${{ github.event.pull_request.number }}..." curl -s -X DELETE \ -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ -H "Accept: application/vnd.github.v3+json" \ "https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/labels/run-ci" ================================================ FILE: .github/workflows/trigger-diff-coverage.yml ================================================ name: Manually Trigger Incremental Coverage Test # Use workflow_dispatch to allow manual triggering on: workflow_dispatch: inputs: base_commit: description: 'The base Commit SHA for comparison (the older one)' required: true head_commit: description: 'The head Commit SHA to generate coverage for (the newer one)' required: true jobs: trigger-circleci-diff-coverage: runs-on: ubuntu-latest steps: - name: "Trigger CircleCI Incremental Coverage Pipeline via API" run: | echo "Triggering CircleCI pipeline for base=${{ github.event.inputs.base_commit }} and head=${{ github.event.inputs.head_commit }}..." curl --request POST \ --url "https://circleci.com/api/v2/project/github/secretflow/scql/pipeline/run" \ --header "Content-Type: application/json" \ --header "Circle-Token: ${{ secrets.CCI_TOKEN }}" \ --data '{"definition_id":"f0835e5e-5836-4869-bcb6-f51ff967281d","checkout":{"branch":"${{ github.event.inputs.head_commit }}"},"parameters":{"base_commit":"${{ github.event.inputs.base_commit }}"},"config":{"branch":"${{ github.event.inputs.head_commit }}"}}' ================================================ FILE: .github/workflows/whitespace-check.yml ================================================ name: Whitespace Check on: pull_request: types: [opened, synchronize] concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: check-whitespace: runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v4 with: fetch-depth: 0 - name: Check for whitespace issues run: | ./scripts/check-whitespace.sh \ "${{ github.event.pull_request.base.sha }}" \ "$GITHUB_STEP_SUMMARY" \ "https://github.com/${{ github.repository }}" ================================================ FILE: .github/workflows/yaml-linter.yml ================================================ --- name: Yaml Lint on: push: branches: - main pull_request: branches: - main permissions: contents: read jobs: yaml-linter: uses: secretflow/.github/.github/workflows/yaml-linter.yml@main ================================================ FILE: .gitignore ================================================ # docs docs/_build # bazel /bazel-* # go bin/* tool-bin/* /vendor # cloudide .cloudide/ /compile_commands.json # clangd cache /external /.cache/ .vscode/launch.json # Translations *.mo *.pot *.pyc .venv # 排除 ide 文件 .idea/ # 排除mac本地文件 *.DS_Store # ignore bazelrc for remote cache .remote.bazelrc # python binding build python/build/ python/package/ ================================================ FILE: .golangci.yml ================================================ # This file contains all available configuration options # with their default values. version: "2" # options for analysis running run: # default concurrency is a available CPU number concurrency: 4 # timeout for analysis, e.g. 30s, 5m, default is 1m timeout: 10m # exit code when at least one issue was found, default is 1 issues-exit-code: 1 # include test files or not, default is true tests: true # list of build tags, all linters use it. Default is empty list. build-tags: [] # output configuration options output: # sort order sort-order: - linter - severity - file # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number" formats: text: path: stdout print-linter-name: true print-issued-lines: true colors: true issues: max-same-issues: 0 max-issues-per-linter: 0 formatters: enable: - gofmt - goimports settings: gofmt: # simplify code: gofmt with `-s` option, true by default simplify: true goimports: # put imports beginning with prefix after 3rd-party packages; # it's a comma-separated list of prefixes local-prefixes: - github.com/secretflow/scql linters: exclusions: paths: - _test\.go - pkg/planner - pkg/parser - pkg/proto-gen - pkg/expression - pkg/types - pkg/util rules: - text: 'shadow: declaration of "(err|ctx)" shadows declaration at' linters: [govet] settings: staticcheck: # All supported checks can be enabled with "all". # To disable checks, prefix them with a minus sign. # Example: [ "all", "-SA1000", "-SA1001"] checks: ["all", "-ST1000", "-ST1003", "-ST1016", "-ST1020", "-ST1021", "-ST1022", "-QF1003", "-QF1006", "-QF1008"] gosec: excludes: - G115 # Potential integer overflow when converting between integer types - G401 # Detect the usage of MD5 or SHA1 - G402 # Look for bad TLS connection settings - G501 # Import blocklist: crypto/md5 revive: rules: - name: unused-parameter # disable unused-parameter rule disabled: true errcheck: # report about not checking of errors in type assetions: `a := b.(MyStruct)`; # default is false: such cases aren't reported by default. check-type-assertions: false # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`; # default is false: such cases aren't reported by default. check-blank: false exclude-functions: - fmt.Fprintf(os.Stdout) - fmt.Fprintln(os.Stdout) - (*database/sql.Rows).Close # prefix is required - (*os.File).Close - (io.ReadCloser).Close - (io.Close).Close - (*google.golang.org/grpc.ClientConn).Close govet: # report about shadowed variables enable: - shadow gocyclo: # minimal code complexity to report, 30 by default (but we recommend 10-20) min-complexity: 10 dupl: # tokens count to trigger issue, 150 by default threshold: 150 goconst: # minimal length of string constant, 3 by default min-len: 3 # minimal occurrences count to trigger, 3 by default min-occurrences: 3 depguard: rules: main: deny: - pkg: "github.com/davecgh/go-spew/spew" misspell: # Correct spellings using locale preferences for US or UK. # Default is to use a neutral variety of English. # Setting locale to US will correct the British spelling of 'colour' to 'color'. locale: US ignore-rules: - someword lll: # max line length, lines longer will be reported. Default is 120. # '\t' is counted as 1 character by default, and can be changed with the tab-width option line-length: 120 # tab width in spaces. Default to 1. tab-width: 1 unused: parameters-are-used: true exported-fields-are-used: true unparam: # Inspect exported functions, default is false. Set to true if no external program/library imports your code. # XXX: if you enable this setting, unparam will report a lot of false-positives in text editors: # if it's called for subdir of a project it can't find external interfaces. All text editor integrations # with golangci-lint call it on a directory with the changed file. check-exported: false nakedret: # make an issue if func has more lines of code than this setting and it has naked returns; default is 30 max-func-lines: 30 prealloc: # XXX: we don't recommend using this linter before doing performance profiling. # For most programs usage of prealloc will be a premature optimization. # Report preallocation suggestions only on simple loops that have no returns/breaks/continues/gotos in them. # True by default. simple: true range-loops: true # Report preallocation suggestions on range loops, true by default for-loops: false # Report preallocation suggestions on for loops, false by default gocritic: # Which checks should be disabled; can't be combined with 'enabled-checks'; default is empty disabled-checks: - regexpMust # Enable multiple checks by tags, run `GL_DEBUG=gocritic golangci-lint` run to see all tags and checks. # Empty list by default. See https://github.com/go-critic/go-critic#usage -> section "Tags". enabled-tags: - performance settings: # settings passed to gocritic captLocal: # must be valid enabled check name paramsOnly: true enable: - unused - govet - revive - errcheck - goconst - dupl # - gosec - staticcheck - bodyclose default: none ================================================ FILE: .licenserc.yaml ================================================ header: license: spdx-id: Apache-2.0 copyright-owner: Ant Group Co., Ltd. copyright-year: auto software-name: secretflow pattern: | Licensed under the Apache License, Version 2.0 \(the "License"\); you may not use this file except in compliance with the License. You may obtain a copy of the License at http[s]?://www\.apache\.org/licenses/LICENSE-2\.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. paths: - "**" paths-ignore: - ".aci" - ".ci" - ".circleci" - ".github" - ".vscode" - "benchmark/docker-compose/broker" - "benchmark/docker-compose/engine" - "benchmark/docker-compose/mysql" - "benchmark/testdata" - "contrib/agent/kusciastub/mock_kusciaapi/mock_kusciaapi.go" - "docs/_static" - "docs/imgs" - "examples" - "pkg/executor/engine_stub_mock.go" - "pkg/expression" - "pkg/infoschema" - "pkg/parser" - "pkg/planner/core" - "pkg/planner/property" - "pkg/planner/util/path.go" - "pkg/privilege" - "pkg/sessionctx" - "pkg/table" - "pkg/types" - "pkg/util/chunk" - "pkg/util/codec" - "pkg/util/execdetails" - "pkg/util/hack" - "pkg/util/math" - "pkg/util/mathutil" - "pkg/util/misc.go" - "pkg/util/mvmap" - "pkg/util/plancodec" - "pkg/util/ranger" - "pkg/util/stringutil" - "pkg/util/testleak" - "pkg/util/testutil" - "pkg/util/texttree" - "scripts" - "**/*.template" - "**/*.patch" - "**/*.pb.go" - "**/*.md" - "**/*.json" - "**/*.yaml" - "**/*.yml" - "**/*.txt" - "**/*.Dockerfile" - "**/Dockerfile" - "**/*.po" - "**/*.svg" - "**/*.bat" - "**/*.rst" - "**/*.patch" - "**/*.csv" - "**/*.tmpl" - "LICENSE" - "NOTICE" - "MODULE.bazel.lock" - ".bazeliskrc" - ".bazelrc" - ".clang-format" - ".clang-tidy" - "**/.gitignore" - "**/.env" - "go.mod" - "go.sum" - "**/Makefile" comment: never license-location-threshold: 80 language: Starlark: extensions: - ".bazel" - ".bazelrc" - "BUILD" - ".bzl" - "WORKSPACE" comment_style_id: PythonStyle Cpp: extensions: - ".cc" - ".h" - ".cu" comment_style_id: DoubleSlash Python: extensions: - ".py" comment_style_id: PythonStyle ================================================ FILE: .markdownlint.yaml ================================================ # Copyright 2025 Ant Group Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Enable all markdownlint rules by default default: true MD013: false # line-length Line length MD024: false # no-duplicate-heading Multiple headings with the same content # List formatting: # # 1. Lists must have a blank line before AND after the list. # 2. Lists start aligned to the left (do not indent the top level list items). # NOTE: markdownlint currently checks indentation for unordered lists only. # Please manually verify that your ordered lists are not indented. # See https://github.com/DavidAnson/markdownlint/issues/138. # 3. You may use one or zero blank lines between list items. # 4. Nested list items should be indented to align with the first character of # the first line. For bullet lists, that means 2 spaces. For numbered # lists, that's 3 spaces (but 4 spaces is okay if that's easier). # 5. In multiline list items, subsequent lines are indented by 2 spaces. # This is not checked automatically, so we're documenting this convention # to make sure the codebase stays consistent. # # Examples: # # * This is a list item that has multiple # lines and each line aligns with the text from the first line. # * This is a nested list, also aligned with the first line. # # For ordered lists, that means three spaces for wrapped lines: # # 1. This is an ordered list item. # 1. The nested list aligns with the first line. ul-indent: indent: 2 # Allow inline HTML no-inline-html: false # Allow dupe heading names only if they're not siblings no-duplicate-heading: siblings_only: true # Allow images w/o alt-text no-alt-text: false ================================================ FILE: .pre-commit-config.yaml ================================================ repos: - repo: https://github.com/gitleaks/gitleaks rev: v8.24.0 hooks: - id: gitleaks - repo: https://github.com/golangci/golangci-lint rev: v2.0.0 hooks: - id: golangci-lint - repo: https://github.com/jumanjihouse/pre-commit-hooks rev: 3.0.0 hooks: - id: shellcheck - repo: https://github.com/pocc/pre-commit-hooks rev: v1.3.5 hooks: - id: cpplint - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/pylint-dev/pylint rev: v2.17.2 hooks: - id: pylint ================================================ FILE: .vscode/cspell.json ================================================ // cSpell Settings { // Version of the setting file. Always 0.2 "version": "0.2", // language - current active spelling language "language": "en", // words - list of words to be always considered correct "words": [ "brokerctl", "gorm", "Kuscia", "kusciaapi", "protojson", "quickstart", "scql", "scqlengine", "secretflow", "spu", "varchar" ] } ================================================ FILE: .vscode/extensions.json ================================================ { "recommendations": [ "BazelBuild.vscode-bazel", "eamodio.gitlens", "golang.go", "ms-python.python", "ms-vscode.cpptools", "jgclark.vscode-todo-highlight", "ms-vscode-remote.remote-ssh", "drblury.protobuf-vsc", "llvm-vs-code-extensions.vscode-clangd", "lextudio.restructuredtext", "bufbuild.vscode-buf" ] } ================================================ FILE: .vscode/settings.json ================================================ { "editor.formatOnSave": true, "gopls": { "formatting.local": "github.com/secretflow/scql" }, "[python]": { "editor.defaultFormatter": "ms-python.black-formatter" }, "files.watcherExclude": { "**/.git/**": true, "**/.cache/**": true, "**/bazel-*/**": true, "**/external/**": true }, "files.insertFinalNewline": true, "files.trimTrailingWhitespace": true, "[proto]": { "editor.defaultFormatter": "bufbuild.vscode-buf", "editor.formatOnSave": true } } ================================================ FILE: BUILD.bazel ================================================ # Copyright 2023 Ant Group Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: CHANGELOG.md ================================================ # Changelog All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Types of changes `Added` for new features. `Changed` for changes in existing functionality. `Deprecated` for soon-to-be removed features. `Removed` for now removed features. `Fixed` for any bug fixes. `Security` in case of vulnerabilities. ## Staging ## [2.0.0] - 2026-03-16 ### Added - Introduced SCQL 2.0 OpenCore architecture: new `CompilerService` gRPC API (`api/v1alpha1/compiler.proto`) with `CompileSQL` endpoint, and a new Go compiler package (`pkg/interpreter/compiler/`) replacing the old interpreter pipeline. - Added Perfetto-based tracing support in engine, controlled by `--enable_trace` and `--trace_log_path` flags. - Added `GroupSecretSum` and `GroupSecretAvg` operators using SPU secret-sharing, replacing the old HE-based group aggregation. Benchmarks show the new algorithm is 5-10x faster. - Added planner optimization rules: correlated subquery decorrelation, group-by threshold enforcement, `JOIN` reorder by party code, and consecutive selection merging. - Added `Rr22Mode` (low/fast) for PSI algorithm negotiation. - Added `opencore-demo` example and quickstart documentation for getting started with the new architecture. ### Changed - **breaking**: Redesigned system architecture from Broker-based (P2P/Centralized) to native Compiler + Engine integration. - **breaking**: `Executor.RunExecutionPlan()` now returns `*scql.QueryResponse` instead of `*scql.SCDBQueryResultResponse`. - Refactored engine operators to use typed `ExecContext` accessor methods (`GetInputTensor()`, `SetOutputTensor()`, etc.) instead of manual tensor table lookups. - Refactored session negotiation: new `Session::Negotiate()` method handles streaming options, PSI options, and curve types via protobuf-serialized `NegotiationOptions`. - Consolidated `StreamingOptions` into `SessionOptions`. - Extracted `RunPlanCore` from `EngineServiceImpl` into a standalone function for reuse in both RPC and task modes. - Updated documentation to reflect the OpenCore architecture. ### Removed - Removed SCDB centralized query server (`pkg/scdb/`, `cmd/scdbserver/`, `cmd/scdbclient/`, `api/scdb_api.proto`). - Removed Broker P2P coordination layer (`pkg/broker/`, `cmd/broker/`, `cmd/brokerctl/`, `api/broker.proto`). - Removed CCL (Column Control Language) authorization mechanism (`api/ccl.proto`, `pkg/interpreter/ccl/`). - Removed old interpreter/translator pipeline (`pkg/interpreter/translator/`, `pkg/interpreter/interpreter.go`, `pkg/interpreter/svc/`). - Removed `cmd/regtest/`, `cmd/agent/`, `cmd/scqltool/`, `pkg/privilege/`, `pkg/executor/job_watcher.go`. - Removed `GroupHESum` operator (replaced by `GroupSecretSum`/`GroupSecretAvg`). - Removed all legacy deployment guides, API references, CCL docs, and CI infrastructure for removed components. ### Fixed - Fixed communication stats not accounting for initialization-phase network traffic. - Fixed potential async task lifecycle issues in `RunPlanSync` by switching from `std::async` to direct invocation. - Fixed null handling in window function, placing null values at the end. - Fixed missing `DESC` keyword when rebuilding ORDER BY statement. - Fixed interpreter creation time losing timezone information. ## [0.9.4] - 2025-07-29 ### Added - Added Archive API to support project archiving. - Enhanced time-type data processing capability: support `STR_TO_DATE` function and implicit conversion from string to time types. - Supported richer expression of statements: `PERCENTILE_DISC`, `BETWEEN AND`, `REPLACE`, etc. ### Changed - Optimized data source reading, improving streaming processing capabilities. - Optimized `JOIN` process to eliminate the need for additional `PLAINTEXT_AFTER_JOIN` CCL for non-result-receiving parties on join keys. ### Fixed - Resolved the issue with compare subquery exceptions in aggregation scenarios. - Fixed column disorder issues in the project tables. - Resolved problems in LogicalOptimizer to prevent the removal of LogicalProjection nodes that could lead to performance and Tensor property inference issues. ## [0.9.3] - 2025-03-07 ### Added - Support for datasource `Doris 2.1.7`. - Support `PERCENT_RANK` window function. - Support various string-related single-party operators, including `UPPER`, `LOWER`, `SUBSTRING`, `TRIM`, `CONCAT` and others. - Support `Scalar Subquery`, the subquery in the right is scalar value, e.g. SELECT * FROM ta JOIN tb ON ta.ID = tb.ID WHERE ta.salary > (SELECT AVG(ta.salary) FROM ta). - Support `Compare Subquery`, allows comparison with ANY or ALL of the subquery results, e.g. SELECT * FROM ta JOIN tb ON ta.ID = tb.ID WHERE ta.salary > ANY(SELECT ta.salary FROM ta), However, comparisons using = or != are not supported in the HAVING clause. For instance, HAVING SUM(ta.salary) = ANY(SELECT salary FROM ta) is not supported. ### Changed - Improved `JOIN` and `IN` performance in streaming mode. - Implemented a more reliable `secret join algorithm`(only works in SEMI2K protocol) inspired by [Scape](https://ieeexplore.ieee.org/document/9835540/). - Optimized the column pruning rule for Join, Selection, and Window nodes in the Logical Optimizer to more effectively remove redundant columns. ### Fixed - Restricted access to SCQLEngine metrics using additional paths like "engine_ip:engine_port/metrics/additional/path". - Prevented creation of tables with the same ref_table name but different db_type - Fixed job creation error when selecting 'OPRF-PSI' but 'server hint' was missing. ## [0.9.2] - 2024-12-23 ### Added - Enhancement: Support `JOIN` after `UNION` operation. - Add SCQL Agent to facilitate running SCQL query tasks in Kuscia, making it easier to integrate into SecretPad. - Support writing results into multi-parties via `SELECT INTO OUTFILE` syntax. - Support datasource `ODPS` via integrating with [dataproxy](https://github.com/secretflow/dataproxy). - Support `order by`. - Support a lot of single-party operators, such as `ABS`, `ASIN`, `EXP`, `FLOOR`, `SQRT` etc. ### Changed - Improve the `JOIN` and `IN` performance via integrating [RR22 PSI](https://github.com/secretflow/psi/blob/v0.5.0b0/psi/proto/psi_v2.proto#L62). - Improve the aggregation with group by performance if `reveal_group_count` enabled. ### Fixed - Fixed an occasional crash issue when canceling query job. - Fixed `select now()` is not supported issue. ## [0.9.1] - 2024-10-16 ### Added - Support window function `ROW_NUMBER()` with PARTITION BY clause and ORDER BY clause. - Add new CCL constraint `REVAL_RANK`. - Add ExplainQuery API with path `/intra/query/explain`. - Support `INSERT INTO SELECT` syntax to allow writing query result back to db (mysql/sqlite/postgres). - Support `trim` function. ### Changed - Improved the job watcher to work better in broker clustered mode. ## [0.9.0] - 2024-08-01 ### Added - Support write outfile to OSS/MINIO via `select into` query. - Support `sin`, `cos`, `acos` function. - Support `geodist` function. - Broker support using postgres as metadata storage. ### Changed - Reduce the memory peak of large-scale intersection tasks through streaming execution. - Link tcmalloc to solve the problem of memory increase. ### Fixed - Fix crashes when dumpfile exceeds 2GB string column. - Reduce the probability of graph checksum inconsistency issues. ## [0.8.1] - 2024-07-02 ### Added - Support session-based log isolation functioality in the SCQL Engine. - Support consul-based broker registration/discovery services, providing ACL/TLS authentication. ### Changed ### Fixed ## [0.8.0] - 2024-06-12 ### Added - Enhanced `FetchResult` RPC and `brokerctl get result` command to report job progress when result is not ready. - Support project/query level configs - Support NULL for private data: including Arithmetic, Logic, Aggregation, etc., {IS [NOT] NULL, IFNULL, COALESCE} are also supported. - Support port isolation for engine link service and control panel service (RunExecutionPlan). - Add new CCL constraint `PLAINTEXT_AS_JOIN_PAYLOAD`. ### Changed - **breaking**: The response value type of Broker API `DoQuery` and `FetchResult` have incompatible changes. ### Fixed ## [0.7.0] - 2024-05-14 ### Added - Added CheckAndUpdate API for self-recovery when status is inconsistent in P2P mode. ### Fixed - Fixed the problem that Broker was unable to detect SCQLEngine crashes or being killed by OOM. ## [0.6.0] - 2024-04-15 ### Added - Support for RSA key pairs in SCQLBroker. - Support running on [kuscia](https://github.com/secretflow/kuscia) and scheduling SCQLEngine dynamic via kuscia job. - Added `dry_run` parameter in DoQuery request, it could be used to check query syntax and CCL without actually executing the query. - Improve Broker high availability, support deploying in multi-node cluster deployment. - Support reading csv from OSS/MINIO. ### Changed - **breaking**: Reshape column data type, data type `LONG` is deprecated. - **breaking**: Modify table schema in broker storage for P2P mode. ## [0.5.0] - 2024-01-10 ### Added - Added support for HTTP data source router. ### Changed - **breaking**: Add table **members** in broker storage for P2P mode. - Speed up GROUP BY with Radix Sort. - Adjusted configuration items for SCQLEngine and SCQLBroker. ### Fixed - Fixed check for grant ccl in P2P mode. ## [0.4.0] - 2023-11-15 ### Added - Added support for P2P mode, no longer need to rely on a trusted third party. - Added support for {datetime, timestamp} data types, as well as related operations. - Support using ArrowSQL as a data source for Engine - Added support for {Limit Cast Mod} operators. ### Changed - Polished document outline. ## [0.3.0] - 2023-09-10 ### Added - Optimize SCQLEngine memory usage, release unused tensors immediately. - Added warning information to the query result. - Added support for {LEFT JOIN, RIGHT JOIN, CASE WHEN, IF} operators ### Changed - Speed up GROUP BY with HEU in some scenarios. - Optimized to support billion-level PSI scenarios. - Drop GRM from SCQL awareness. We extend the syntax of create user statement and modify the syntax of create table statement. - Used json string format to configure spu runtime in scdb yaml conf. - Speed up JOIN, IN with Unbalanced PSI in scenarios with unbalanced data. ## [0.2.0] - 2023-06-30 ### Added - Added support for union operator. - Added support for reading CSV files as a data source for Engine. - Added support for using PostgreSQL as a database for Engine. - Added support for change password with ALTER USER statement. - Added support for removing table-level and database-level permissions with the REVOKE statement. - Added support for structured audit log. - Added support for the float64 data type in the Engine. - Added the Chinese documentation. ### Changed - Change some description in document. - Enrich test cases. - Enhanced support for security protocols of Cheetah and ABY3. - Optimized GROUP BY logic. - Optimized execution plan nodes. - Optimized the execution logic of the runSQL. - Optimized the three party ccl in join node. ### Fixed - Fixed create database failed [#19](https://github.com/secretflow/scql/issues/19). - Fixed not support group by string[#48](https://github.com/secretflow/scql/pull/48). ## [0.1.0] - 2023-03-28 ### Added - SCQL init release ================================================ FILE: CONTRIBUTING.md ================================================ # Contributing ## Contributor License Agreement Contributions to this project must be accompanied by a Contributor License Agreement. You (or your employer) retain the copyright to your contribution; this simply gives us permission to use and redistribute your contributions as part of the project. ## Repo layout - Please see [repo layout](REPO_LAYOUT.md). ## Style ### Go coding style Go code follows [Uber Go Style Guide](https://github.com/uber-go/guide/blob/master/style.md) ### C++ coding style In general, please use clang-format to format code, and follow clang-tidy tips. Most of the code style is derived from the [Google C++ style guidelines](https://google.github.io/styleguide/cppguide.html), except: - Exceptions are allowed and encouraged where appropriate. - Header guards should use `#pragma once`. ### Other tips - Git commit message should be meaningful, we suggest imperative [keywords](https://github.com/joelparkerhenderson/git_commit_message#summary-keywords). - Developer must write unit-test (line coverage must be greater than 80%), tests should be deterministic. ## Build ### Prerequisite #### Docker ```sh ## start dev container docker run -d -it --name scql-dev-$(whoami) \ --mount type=bind,source="$(pwd)",target=/home/admin/dev/ \ -w /home/admin/dev \ --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \ --cap-add=NET_ADMIN \ --privileged=true \ secretflow/scql-ci:latest /bin/bash # attach to dev container docker exec -it scql-dev-$(whoami) bash ``` ### Build & UnitTest ```sh # build SCQL engine as release bazelisk build //engine/exe:scqlengine -c opt # run unittests for SCQL engine bazelisk test //engine/... # build with address sanitizer bazelisk build --config=asan //engine/exe:scqlengine # build go code make # run go unit tests go test ./pkg/... ``` ### Build docs ```sh # prerequisite pip3 install -U -r docs/requirements.txt # Build HTML docs, and the result is placed in directory 'docs/_build/html' # Build documentation in English and Chinese make doc ``` ================================================ FILE: LEGAL.md ================================================ # Legal Disclaimer Within this source code, the comments in Chinese shall be the original, governing version. Any comment in other languages are for reference only. In the event of any conflict between the Chinese language version comments and other language version comments, the Chinese language version shall prevail. 法律免责声明 关于代码注释部分，中文注释为官方版本，其它语言注释仅做参考。中文注释可能与其它语言注释存在不一致，当中文注释与其它语言注释存在不一致时，请以中文注释为准。 ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: MODULE.bazel ================================================ # Copyright 2024 Ant Group Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. module( name = "scql", version = "0.9.5dev", compatibility_level = 1, ) bazel_dep(name = "yacl", version = "0.4.5b12-nightly-20251024") bazel_dep(name = "psi", version = "0.6.0.dev250922") bazel_dep(name = "spulib", version = "0.9.5rc1") git_override( module_name = "spulib", commit = "2387c999a4bd2b7e85072f4e25a173a60b5958d4", remote = "https://github.com/secretflow/spu.git", strip_prefix = "src", ) bazel_dep(name = "dataproxy_sdk_cc", version = "0.5.0.dev20250603") bazel_dep(name = "kuscia", version = "0.13.0b1") bazel_dep(name = "apple_support", version = "1.24.5") bazel_dep(name = "rules_cuda", version = "0.2.3") bazel_dep(name = "rules_cc", version = "0.0.12") bazel_dep(name = "rules_proto", version = "6.0.2") # non module dependencies non_module_dependencies = use_extension("//bazel:defs.bzl", "non_module_dependencies") use_repo( non_module_dependencies, "com_github_duckdb", "com_github_gperftools_gperftools", "org_pocoproject_poco", ) bazel_dep(name = "arrow", version = "17.0.0.bcr.1", repo_name = "org_apache_arrow") bazel_dep(name = "ncurses", version = "6.4.20221231.bcr.3") bazel_dep(name = "rules_pkg", version = "1.0.1") bazel_dep(name = "rules_java", version = "8.6.1") bazel_dep(name = "rules_go", version = "0.53.0") bazel_dep(name = "rules_proto_grpc_cpp", version = "5.0.1") bazel_dep(name = "rules_foreign_cc", version = "0.15.0") single_version_override( module_name = "rules_foreign_cc", patch_strip = 1, patches = [ "//bazel/patches:rules_foreign_cc.patch", ], version = "0.15.0", ) bazel_dep(name = "bazel_features", version = "1.20.0") bazel_dep(name = "platforms", version = "0.0.8") bazel_dep(name = "openssl", version = "3.3.2.bcr.1") bazel_dep(name = "spdlog", version = "1.14.1") bazel_dep(name = "fmt", version = "11.0.2") bazel_dep(name = "brpc", version = "1.13.0") bazel_dep(name = "abseil-cpp", version = "20240722.0") bazel_dep(name = "boost.uuid", version = "1.83.0.bcr.1") bazel_dep(name = "boost.multiprecision", version = "1.83.0") bazel_dep(name = "boost.endian", version = "1.83.0") bazel_dep(name = "boost.geometry", version = "1.83.0") bazel_dep(name = "boost.graph", version = "1.83.0") bazel_dep(name = "boost.spirit", version = "1.83.0") bazel_dep(name = "bazel_skylib", version = "1.7.1") bazel_dep(name = "boost.serialization", version = "1.83.0.bcr.1") bazel_dep(name = "prometheus-cpp", version = "1.2.4") bazel_dep(name = "protobuf", version = "27.3", repo_name = "com_google_protobuf") single_version_override( module_name = "protobuf", version = "27.3", ) bazel_dep(name = "perfetto", version = "41.0") bazel_dep(name = "googleapis", version = "0.0.0-20240819-fe8ba054a") bazel_dep(name = "zlib", version = "1.3.1.bcr.3") bazel_dep(name = "rules_python", version = "0.34.0") bazel_dep(name = "gflags", version = "2.2.2") bazel_dep(name = "msgpack-c", version = "6.1.0") bazel_dep(name = "grpc", version = "1.66.0.bcr.4") bazel_dep(name = "pybind11_bazel", version = "3.0.0") single_version_override( module_name = "grpc", version = "1.66.0.bcr.4", ) single_version_override( module_name = "openssl", version = "3.3.2.bcr.1", ) # use sf.bcr bazel_dep(name = "curl", version = "8.4.0.bcr.2") single_version_override( module_name = "curl", version = "8.4.0.bcr.2", ) new_local_repository = use_repo_rule("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") new_local_repository( name = "macos_omp_x64", build_file = "@yacl//bazel:local_openmp_macos.BUILD", path = "/usr/local/opt/libomp", ) new_local_repository( name = "macos_omp_arm64", build_file = "@yacl//bazel:local_openmp_macos.BUILD", path = "/opt/homebrew/opt/libomp/", ) python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain( ignore_root_user_error = True, python_version = "3.11", ) # test bazel_dep(name = "googletest", version = "1.15.2", dev_dependency = True) bazel_dep(name = "google_benchmark", version = "1.8.5", dev_dependency = True) bazel_dep(name = "rules_buf", version = "0.4.0") ================================================ FILE: Makefile ================================================ export GO111MODULE=on GOPATH := ${GOPATH}:${PWD} TOOLBIN := ${PWD}/tool-bin export PATH := ${TOOLBIN}:$(PATH) export GOFLAGS=-buildmode=pie -buildvcs=false export CGO_CPPFLAGS=-fstack-protector-strong -D_FORTIFY_SOURCE=2 UNAME_S := $(shell uname -s) ifeq ($(UNAME_S),Linux) export CGO_LDFLAGS=-Wl,-z,relro,-z,now,-z,noexecstack endif .PHONY: clean vet lint test detect-shadowing fast pb prepare fmt gogenerate default: install install: clean prepare fmt vet gogenerate GOBIN=${PWD}/bin go install -ldflags "-X main.version=${SCQL_VERSION}" ./cmd/... gogenerate: go generate ./pkg/... go generate ./cmd/... fast: fmt vet GOBIN=${PWD}/bin go install ./cmd/... parser: cd pkg/parser && make binary: clean prepare fmt vet gogenerate $(eval SCQL_VERSION := $(shell bash ${PWD}/version_build.sh)) echo "Binary version: ${SCQL_VERSION}" GOBIN=${PWD}/bin go install -ldflags "-X main.version=${SCQL_VERSION}" ./cmd/... bazelisk --host_jvm_args=-Xmx8g build //engine/exe:scqlengine -c opt --jobs=32 bash ${PWD}/version_build.sh -r binary-cov: clean prepare fmt vet gogenerate $(eval SCQL_VERSION := $(shell bash ${PWD}/version_build.sh)) echo "Binary version: ${SCQL_VERSION}" GOBIN=${PWD}/bin go install -ldflags "-X main.version=${SCQL_VERSION}" -cover ./... bazelisk --host_jvm_args=-Xmx8g build //engine/exe:scqlengine --jobs=32 --collect_code_coverage --instrumentation_filter=//engine/... --noremote_upload_local_results --copt=-DNDEBUG bash ${PWD}/version_build.sh -r pb: clean $(RM) -rf pkg/proto-gen/* ./api/generate_proto.sh fmt: go fmt ./pkg/... vet: go vet -unsafeptr=false ./pkg/... doc: go run ./cmd/docgen/main.go cd docs && rm -rf _build && make build lint: GOLINT-exists -${TOOLBIN}/golangci-lint run --out-format=colored-line-number detect-shadowing: go vet -vettool=$(shell which shadow) -strict ./... clean: $(RM) bin/* $(RM) *.coverprofile test: go test -v -cover ./pkg/... testsum: go run gotest.tools/gotestsum@latest ./pkg/... coverage: install go list -f '{{if gt (len .TestGoFiles) 0}}"go test -covermode count -coverprofile {{.Name}}.coverprofile -coverpkg ./... {{.ImportPath}}"{{end}}' ./... | xargs -I {} bash -c {} find . -name "*.coverprofile" $(info Use `go tool cover -html MODULE_NAME.coverprofile`) prepare: GO-exists GO-package GO-exists: $(if $(shell command -v go 2> /dev/null),$(info Found `go`),$(error Please install go (prefer v1.22): refer to `https://golang.org/dl/`)) go version go env GOPROXY GOLINT-exists: $(if $(shell command -v golangci-lint 2> /dev/null),$(info Found `golangci-lint`),$(shell curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b ${TOOLBIN} v1.61.0)) GO-package: @GOBIN=${TOOLBIN} go install go.uber.org/mock/mockgen@latest && \ GOBIN=${TOOLBIN} go install golang.org/x/tools/cmd/goyacc@latest && \ GOBIN=${TOOLBIN} go install github.com/mattn/goveralls@latest && \ GOBIN=${TOOLBIN} go install github.com/rakyll/gotest@latest && \ GOBIN=${TOOLBIN} go install golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow@latest ================================================ FILE: README.md ================================================ # SCQL [![CircleCI](https://dl.circleci.com/status-badge/img/gh/secretflow/scql/tree/main.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/secretflow/scql/tree/main) Secure Collaborative Query Language (SCQL) is a system that translates SQL statements into Secure Multiparty Computation (SMC) primitives and executes them on a federation of database systems. > **Note**: If you are looking for SCQL 1.0, please check the [1.x branch](https://github.com/secretflow/scql/tree/release/1.x). ![SCQL Workflow](./docs/imgs/scql_workflow.png) ## Documentation - [Documentation in English](https://www.secretflow.org.cn/en/docs/scql) - [中文文档](https://www.secretflow.org.cn/zh-CN/docs/scql/) ## Docker Image Release - Official release docker image: [secretflow/scql](https://hub.docker.com/r/secretflow/scql/tags) - We also have images at Alibaba Cloud: secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/scql:[tag] ## Contribution Guidelines If you would like to contribute to SCQL, please see the [Contribution guidelines](CONTRIBUTING.md). This documentation also contains instructions for [build and testing](CONTRIBUTING.md#build) ## Hardware Requirements The following requirements only apply to SCQLEngine. - CPU - x86_64: minimum required AVX instruction set. For FourQ based PSI, the AVX2 instruction set is required. ## Disclaimer Non-release versions of SCQL are prohibited to use in any production environment due to possible bugs, glitches, lack of functionality, security issues or other problems. ## Citing SCQL If you think SCQL is helpful for your research or development, please consider citing our [paper](https://www.vldb.org/pvldb/vol17/p3987-fang.pdf): ```text @article{scql, author = {Wenjing Fang and Shunde Cao and Guojin Hua and Junming Ma and Yongqiang Yu and Qunshan Huang and Jun Feng and Jin Tan and Xiaopeng Zan and Pu Duan and Yang Yang and Li Wang and Ke Zhang and Lei Wang}, title = {SecretFlow-SCQL: {A} Secure Collaborative Query pLatform}, journal = {Proc. VLDB Endow.}, volume = {17}, number = {12}, pages = {3987--4000}, year = {2024}, url = {https://www.vldb.org/pvldb/vol17/p3987-fang.pdf}, } ``` ## Acknowledgments - Thanks [TiDB](https://github.com/pingcap/tidb) for providing a powerful SQL parser and planner. ================================================ FILE: REPO_LAYOUT.md ================================================ # Repository Layout - [api/](api/): SCQL protocol files. - [examples/](examples/): SCQL examples. - [tutorial](examples/tutorial/): SCQL open core tutorial. - [docs/](docs/): Documents of SCQL. - [cmd/](cmd/): Main applications for SCQL. - [docgen/](cmd/docgen/): SCQL operators document generator. - [pkg/](pkg/): SCQL library code. - [constant/](pkg/constant/): Common constant values. - [parser/](pkg/parser/): SCQL parser. - [types/](pkg/types/): SCQL data types. - [table/](pkg/table/): SCQL table info. - [sessionctx/](pkg/sessionctx/): SCQL logical plan context. - [util/](pkg/util/): SCQL utils. - [planner/](pkg/planner/): SCQL planner. - [interpreter/](pkg/interpreter/): SCQL interpreter. - [compiler/](pkg/interpreter/compiler/): SCQL compiler that translates SQL to execution graph. - [graph/](pkg/interpreter/graph/): Execution graph data structures and processing. - [operator/](pkg/interpreter/operator/): SCQL operators. - [sc/](pkg/interpreter/sc/): SCQL compiler Python bindings using gopy. - [executor/](pkg/executor/): DQL executor. It dispatches execution dag to SCQL engine. - [engine/](engine/): SCQL execution engine, implemented in C++. - [exe/](engine/exe/): SCQL execution engine applications. - [services/](engine/services/): Engine RPC services. - [link/](engine/link/): MPI framework based on [YACL link](https://github.com/secretflow/yacl/tree/main/yacl/link). - [core/](engine/core/): Basic data structures used in engine. - [framework/](engine/framework/): Engine framework. - [operator/](engine/operator/): Oblivious operators. - [datasource/](engine/datasource/): SCQL data source adaptors/connectors. - [util/](engine/util/): Engine utilities. ================================================ FILE: SECURITY.md ================================================ # Security If you believe you have found a security vulnerability in any SecretFlow repository that meets [SecretFlow's definition of a security vulnerability](https://security.alipay.com/announcement.htm?id=1), please report it to us as described below. ## Reporting Security Issues **Please do not report security vulnerabilities through public GitHub issues.** Instead, please report them to the ANT GROUP SECURITY Response Center at [https://security.alipay.com/](https://security.alipay.com/). If you prefer to submit without logging in, send email to [antsrc@alipay.com](mailto:antsrc@alipay.com). You should receive a response within 48 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [https://security.alipay.com/](https://security.alipay.com/). Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) * Full paths of source file(s) related to the manifestation of the issue * The location of the affected source code (tag/branch/commit or direct URL) * Any special configuration required to reproduce the issue * Step-by-step instructions to reproduce the issue * Proof-of-concept or exploit code (if possible) * Impact of the issue, including how an attacker might exploit the issue This information will help us triage your report more quickly. ## Preferred Languages We prefer all communications to be in Chinese or English. ================================================ FILE: api/BUILD.bazel ================================================ # Copyright 2023 Ant Group Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. load("@rules_buf//buf:defs.bzl", "buf_lint_test") load("@rules_cc//cc:defs.bzl", "cc_proto_library") load("@rules_go//proto:def.bzl", "go_proto_library") load("@rules_java//java:defs.bzl", "java_proto_library") load("@rules_proto//proto:defs.bzl", "proto_library") package(default_visibility = ["//visibility:public"]) proto_library( name = "core_proto", srcs = ["core.proto"], deps = [ ":common_proto", ":status_proto", ], ) proto_library( name = "status_code_proto", srcs = ["status_code.proto"], ) proto_library( name = "engine_proto", srcs = ["engine.proto"], deps = [ ":common_proto", ":core_proto", ":status_proto", ":subgraph_proto", "@com_google_protobuf//:empty_proto", "@spulib//libspu:spu_proto", ], ) proto_library( name = "status_proto", srcs = ["status.proto"], deps = [ "@com_google_protobuf//:any_proto", ], ) proto_library( name = "common_proto", srcs = ["common.proto"], deps = [ "@com_google_protobuf//:any_proto", "@com_google_protobuf//:timestamp_proto", ], ) cc_proto_library( name = "common_cc_proto", deps = [":common_proto"], ) cc_proto_library( name = "engine_cc_proto", deps = [":engine_proto"], ) cc_proto_library( name = "core_cc_proto", deps = [":core_proto"], ) cc_proto_library( name = "status_cc_proto", deps = [":status_code_proto"], ) cc_proto_library( name = "interpreter_cc_proto", deps = [":interpreter_proto"], ) proto_library( name = "subgraph_proto", srcs = [ "subgraph.proto", ], deps = [":core_proto"], ) proto_library( name = "interpreter_proto", srcs = ["interpreter.proto"], deps = [ ":common_proto", ":core_proto", ":status_proto", ":subgraph_proto", "@com_google_protobuf//:timestamp_proto", "@spulib//libspu:spu_proto", ], ) proto_library( name = "scql_task_proto", srcs = ["scql_task.proto"], deps = [ ":common_proto", ":engine_proto", ":subgraph_proto", "@spulib//libspu:spu_proto", ], ) cc_proto_library( name = "scql_task_cc_proto", deps = [":scql_task_proto"], ) go_proto_library( name = "spu_go_proto", importpath = "github.com/secretflow/scql/pkg/proto-gen/spu", protos = ["@spulib//libspu:spu_proto"], ) go_proto_library( name = "googleapis_go_proto", importpath = "google.golang.org/genproto/googleapis/api", protos = [ "@googleapis//google/api:annotations_proto", "@googleapis//google/api:field_behavior_proto", "@googleapis//google/api:http_proto", ], ) go_proto_library( name = "scql_go_proto", compilers = ["@rules_go//proto:go_grpc"], importpath = "github.com/secretflow/scql/pkg/proto-gen/scql", protos = [ ":common_proto", ":core_proto", ":engine_proto", ":interpreter_proto", ":scql_task_proto", ":status_code_proto", ":status_proto", ":subgraph_proto", ], deps = [ ":googleapis_go_proto", ":spu_go_proto", ], ) # lint buf_lint_test( name = "core_proto_lint", config = "buf.yaml", targets = [":core_proto"], ) buf_lint_test( name = "status_code_proto_lint", config = "buf.yaml", targets = [":status_code_proto"], ) buf_lint_test( name = "engine_proto_lint", config = "buf.yaml", targets = [":engine_proto"], ) buf_lint_test( name = "status_proto_lint", config = "buf.yaml", targets = [":status_proto"], ) buf_lint_test( name = "common_proto_lint", config = "buf.yaml", targets = [":common_proto"], ) buf_lint_test( name = "subgraph_proto_lint", config = "buf.yaml", targets = [":subgraph_proto"], ) buf_lint_test( name = "interpreter_proto_lint", config = "buf.yaml", targets = [":interpreter_proto"], ) ================================================ FILE: api/buf.yaml ================================================ version: v2 modules: - path: . name: buf.build/secretflow/scql-protos deps: - buf.build/googleapis/googleapis - buf.build/protocolbuffers/wellknowntypes lint: use: - DEFAULT - PACKAGE_DIRECTORY_MATCH - COMMENTS - UNARY_RPC except: # Allow direct resource returns for Create/Get operations following Google AIP patterns - RPC_RESPONSE_STANDARD_NAME - RPC_REQUEST_RESPONSE_UNIQUE ignore_only: # You can ignore specific files or rules if needed # - path/to/file.proto breaking: use: - FILE ================================================ FILE: api/common.proto ================================================ // Copyright 2023 Ant Group Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // syntax = "proto3"; package scql.pb; import "google/protobuf/any.proto"; import "google/protobuf/timestamp.proto"; option go_package = "proto-gen/scql"; option java_package = "org.secretflow.scql"; // RequestHeader carries the user custom headers. message RequestHeader { // Custom headers used to record custom information. map custom_headers = 1; } message DebugOptions { bool enable_psi_detail_log = 1; } // (-- TODO: move SQLWarning to a proper place --) message SQLWarning { // Description of the warning string reason = 1; } // IOStats contains input/output statistics for a job, including bytes sent and // received, as well as the number of send and receive actions performed. message IOStats { uint64 send_bytes = 1; uint64 recv_bytes = 2; uint64 send_actions = 3; uint64 recv_actions = 4; } // StageInfo provides information about an individual stage within a running // job. message StageInfo { // The name of the stage. string name = 1; // A brief summary describing the stage string summary = 2; // Stage start time google.protobuf.Timestamp start_time = 3; // Personalized details that may have different structures depending on the // stage type. google.protobuf.Any details = 4; } // JobProgress provides detailed information about the progress of a running // job. message JobProgress { // Job start time google.protobuf.Timestamp start_time = 1; // The total number of stages planned for the job. int32 stages_count = 2; // The number of stages that have been executed so far. int32 executed_stages = 3; IOStats io_stats = 4; // A list of currently running stages, providing insight into which parts of // the job are active. repeated StageInfo running_stages = 5; } enum JobState { JOB_STATE_UNSPECIFIED = 0; JOB_INITIALIZED = 1; JOB_RUNNING = 2; JOB_SUCCEEDED = 3; JOB_FAILED = 4; JOB_CANCELED = 5; } message LinkConfig { int64 link_recv_timeout_sec = 1; int64 link_throttle_window_size = 2; int64 link_chunked_send_parallel_size = 3; int64 http_max_payload_size = 4; } enum PsiAlgorithmType { // auto means choosing psi type by engine AUTO = 0; ECDH = 1; OPRF = 2; RR22 = 3; } enum Rr22Mode { UNDEFINED = 0; LOW_MODE = 1; FAST_MODE = 2; } message PsiConfig { int32 psi_curve_type = 1; PsiAlgorithmType psi_type = 2; // activated when psi_type is rr22 Rr22Mode rr22_mode = 3; } message LogConfig { bool enable_session_logger_separation = 1; } message Placeholder { // placeholder name string name = 1; // placeholder name, e.g. string, int64, float string data_type = 2; } message Placeholders { repeated Placeholder placeholders = 1; } message Variable { string name = 1; string string_data = 2; int64 int64_data = 3; float float_data = 4; bool bool_data = 5; } ================================================ FILE: api/core.proto ================================================ // Copyright 2023 Ant Group Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // syntax = "proto3"; package scql.pb; import "api/common.proto"; import "api/status.proto"; option go_package = "proto-gen/scql"; option java_package = "org.secretflow.scql"; // Defines a tensor shape. A dimension can be either an integer value // or a symbolic variable. A symbolic variable represents an unknown // dimension. message TensorShape { message Dimension { oneof value { int64 dim_value = 1; string dim_param = 2; // shape is unknown. } } repeated Dimension dim = 1; } enum PrimitiveDataType { PrimitiveDataType_UNDEFINED = 0; // Numeric types INT8 = 1; // the 8-bit signed integer type INT16 = 2; // the 16-bit signed integer type INT32 = 3; // the 32-bit signed integer type INT64 = 4; // the 64-bit signed integer type FLOAT32 = 5; // the 32-bit binary floating point type FLOAT64 = 6; // the 64-bit binary floating point type // Other types BOOL = 7; STRING = 8; // DATETIME and TIMESTAMP DATETIME = 9; // https://dev.mysql.com/doc/refman/8.0/en/datetime.html TIMESTAMP = 10; // seconds since '1970-01-01 00:00:00' UTC DECIMAL = 11; } // Tensor options. enum TensorOptions { // A tensor with data. VALUE = 0; // A tensor with reference (URI). REFERENCE = 1; // A tensor variable (declaration). VARIABLE = 2; } enum TensorStatus { // Unknown. TENSORSTATUS_UNKNOWN = 0; // Private. TENSORSTATUS_PRIVATE = 1; // Secret, usually in the form of secret sharing. TENSORSTATUS_SECRET = 2; // Ciphertext, usually in the form of homomorphic encryption ciphertext. TENSORSTATUS_CIPHER = 3; // Public. TENSORSTATUS_PUBLIC = 4; } message TensorAnnotation { TensorStatus status = 1; } // A tensor data representation. message Tensor { // Tensor name. string name = 1; // Tensor shape. // In SCQL cases, it's normally [M] (a vector with M elements). TensorShape shape = 2; // Tensor element type. PrimitiveDataType elem_type = 3; // used by decimal type int32 scale = 14; int32 width = 15; // Tensor options. TensorOptions option = 4; // Tensor annotation carries physical status information. // It MUST be there if the