[ { "path": ".bazeliskrc", "content": "USE_BAZEL_VERSION=7.7.1\n" }, { "path": ".bazelrc", "content": "common --experimental_repo_remote_exec\ncommon --experimental_cc_shared_library\ncommon --experimental_ui_max_stdouterr_bytes=-1\ncommon --experimental_remote_cache_compression\ncommon --experimental_remote_cache_compression_threshold=100\ncommon --nolegacy_important_outputs\n\ncommon --remote_download_regex='.*\\/scqlengine$'\ncommon --registry=https://raw.githubusercontent.com/secretflow/bazel-registry/main\ncommon --registry=https://bcr.bazel.build\n\ncommon --remote_download_outputs=all\n\ncommon --experimental_proto_descriptor_sets_include_source_info\n\nbuild --incompatible_new_actions_api=false\nbuild --copt=-fdiagnostics-color=always\nbuild --enable_platform_specific_config\n\nbuild --cxxopt=-std=c++17\nbuild --host_cxxopt=-std=c++17\nbuild --linkopt -fvisibility=hidden\n# default off CUDA build\nbuild --@rules_cuda//cuda:enable=false\ntest --@rules_cuda//cuda:enable=false\n\n# Binary safety flags\nbuild --copt=-fPIC\nbuild --host_copt=-fstack-protector-strong\nbuild:linux --host_copt=-Wl,-z,noexecstack\nbuild:macos --host_copt=-Wa,--noexecstack\n\n\n\ntest --keep_going\ntest --test_output=errors\ntest --test_timeout=1800\n\n# platform specific config\n# Bazel will automatic pick platform config since we have enable_platform_specific_config set\nbuild:macos --copt=-Xclang=-fopenmp\nbuild:macos --copt=-Wno-unused-command-line-argument\nbuild:macos --features=-supports_dynamic_linker\n# build:macos --cxxopt -Wno-error=missing-template-arg-list-after-template-kw\n# build:macos --cxxopt -Wno-error=vla-cxx-extension\nbuild:macos --macos_minimum_os=13.0\nbuild:macos --host_macos_minimum_os=13.0\nbuild:macos --action_env MACOSX_DEPLOYMENT_TARGET=13.0\n\n# static link libstdc++ & libgcc on Linux\nbuild:linux --copt=-fopenmp\nbuild:linux --linkopt=-fopenmp\nbuild:linux --action_env=BAZEL_LINKOPTS=-static-libstdc++:-static-libgcc\nbuild:linux --action_env=BAZEL_LINKLIBS=-l%:libstdc++.a:-l%:libgcc.a\n\nbuild:asan --strip=never\nbuild:asan --copt -fno-sanitize-recover=all\nbuild:asan --copt -fsanitize=address\nbuild:asan --copt -Og\nbuild:asan --copt -g\nbuild:asan --copt -fno-omit-frame-pointer\nbuild:asan --linkopt -fsanitize=address\nbuild:asan --define disable_tcmalloc=true\nbuild:asan --copt=\"-Wno-error=uninitialized\"\nbuild:asan --copt=\"-Wno-error=maybe-uninitialized\"\nbuild:asan --action_env=ASAN_OPTIONS=detect_odr_violation=0\n" }, { "path": ".circleci/config.yml", "content": "# Copyright 2023 Ant Group Co., Ltd.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nversion: 2.1\n\nsetup: true\n\norbs:\n path-filtering: circleci/path-filtering@2.0.1\n continuation: circleci/continuation@2.0.1\n\nparameters:\n enable_regtest:\n type: boolean\n default: false\n\nworkflows:\n run_ut:\n when:\n not: << pipeline.parameters.enable_regtest >>\n jobs:\n - path-filtering/filter:\n base-revision: main\n config-path: .circleci/lite-unittest-config.yml\n mapping: .circleci/path-filtering/unittest.conf\n\n run_regtest:\n when: << pipeline.parameters.enable_regtest >>\n jobs:\n - continuation/continue:\n configuration_path: .circleci/regtest-config.yml\n" }, { "path": ".circleci/coverage-config.yml", "content": "# Copyright 2025 Ant Group Co., Ltd.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# Use the latest 2.1 version of CircleCI pipeline process engine.\n# See: https://circleci.com/docs/2.0/configuration-reference\n\nversion: 2.1\n\norbs:\n coveralls: coveralls/coveralls@2.2.5\n\ncommands:\n setup_go_test:\n steps:\n - run:\n name: Install gcov2lcov\n command: |\n which gcov2lcov || go install github.com/jandelgado/gcov2lcov@latest\n\n setup_cpp_test:\n steps:\n - run:\n name: \"Checkout devtools\"\n command: git clone --depth=1 https://github.com/secretflow/devtools.git ../devtools\n - run:\n name: Setup BuildBuddy Cache\n command: ../devtools/bazel_cache_setup.py\n\n run_go_test:\n description: \"Run go cov tests\"\n steps:\n - run:\n name: \"Go Cov Test\"\n command: |\n set +e\n go mod tidy\n echo \"Running tests with coverage...\"\n go test -mod=readonly -timeout=30m -v -cover -race -coverprofile=coverage.tmp ./pkg/...\n cat coverage.tmp | grep -v '\\.pb\\.go:\\|_mock\\.go:' > coverage.out\n gcov2lcov -infile=coverage.out -outfile=coverage.lcov\n\n run_cpp_test:\n description: \"Run cpp cov tests and collect artifacts\"\n parameters:\n extra_bazel_args:\n type: string\n default: \"\"\n find_executable_flag:\n type: string\n default: \"-executable\"\n steps:\n - run:\n name: \"Cpp Test\"\n command: |\n set +e\n declare -i test_status\n\n echo \"Running tests with coverage...\"\n bazelisk --host_jvm_args=-Xmx8g coverage //engine/... \\\n << parameters.extra_bazel_args >> \\\n --combined_report=lcov \\\n --jobs=auto \\\n --ui_event_filters=-info,-debug,-warning \\\n --test_output=errors | tee test_result.log\n\n # Capture the exit status of the Bazel command\n test_status=${PIPESTATUS[0]}\n if [ ${test_status} -eq 0 ]; then\n echo \"Processing coverage...\"\n lcov --remove bazel-out/_coverage/_coverage_report.dat '*.pb.h' '*.pb.cc' -o bazel-out/_coverage/_coverage_report_filtered.dat\n else\n echo \"Bazel coverage failed, skipping lcov processing. Archiving binaries and logs...\"\n find bazel-bin/ << parameters.find_executable_flag >> -type f -name \"*_test\" -print0 | xargs -0 tar -cvzf test_binary.tar.gz\n find bazel-testlogs/ -type f -name \"test.log\" -print0 | xargs -0 tar -cvzf test_logs.tar.gz\n fi\n\n sh ../devtools/rename-junit-xml.sh\n exit ${test_status}\n\n # ref: https://support.circleci.com/hc/en-us/articles/14114124583195-How-to-set-a-custom-maximum-job-duration\n cancel_after_timeout:\n description: \"Cancel job if it takes too long\"\n parameters:\n timeout:\n type: string\n default: \"60m\"\n steps:\n - run:\n name: Set maximum job duration to << parameters.timeout >>\n background: true\n command: |\n sleep << parameters.timeout >>\n curl --request POST \\\n --url https://circleci.com/api/v2/project/gh/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/job/$CIRCLE_BUILD_NUM/cancel \\\n --header \"Circle-Token: $CIRCLECI_API_TOKEN\"\n\njobs:\n linux_go_cov:\n docker:\n - image: secretflow/scql-ci:latest\n resource_class: \"large\"\n steps:\n - cancel_after_timeout:\n timeout: \"30m\"\n - checkout\n - setup_go_test\n - run_go_test\n - coveralls/upload:\n coverage_file: coverage.lcov\n coverage_format: lcov\n flag_name: \"go-tests\"\n\n linux_cpp_cov:\n docker:\n - image: secretflow/scql-ci:latest\n resource_class: \"2xlarge\"\n steps:\n - cancel_after_timeout:\n timeout: \"90m\"\n - checkout\n - run:\n name: \"Install lcov\"\n command: apt-get update && apt-get install -y lcov\n - setup_cpp_test\n - run_cpp_test:\n extra_bazel_args: \"-c opt\"\n find_executable_flag: \"-executable\"\n - coveralls/upload:\n coverage_file: bazel-out/_coverage/_coverage_report_filtered.dat\n coverage_format: lcov\n flag_name: \"cpp-tests\"\n - store_test_results:\n path: test-results\n - store_artifacts:\n when: on_fail\n path: test_binary.tar.gz\n - store_artifacts:\n when: on_fail\n path: test_logs.tar.gz\n\nworkflows:\n run_cov:\n jobs:\n - linux_go_cov\n - linux_cpp_cov\n" }, { "path": ".circleci/diff-cover-config.yml", "content": "# Copyright 2025 Ant Group Co., Ltd.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# Use the latest 2.1 version of CircleCI pipeline process engine.\n# See: https://circleci.com/docs/2.0/configuration-reference\n\nversion: 2.1\n\nparameters:\n base_commit:\n type: string\n description: \"The base commit SHA for the diff.\"\n default: \"origin/main\"\n\ncommands:\n setup_go_test:\n steps:\n - run:\n name: Install gcov2lcov\n command: |\n which gcov2lcov || go install github.com/jandelgado/gcov2lcov@latest\n\n setup_cpp_test:\n steps:\n - run:\n name: \"Checkout devtools\"\n command: git clone --depth=1 https://github.com/secretflow/devtools.git ../devtools\n - run:\n name: Setup BuildBuddy Cache\n command: ../devtools/bazel_cache_setup.py\n\n run_go_test:\n description: \"Run go cov tests\"\n steps:\n - run:\n name: \"Go Cov Test\"\n command: |\n set +e\n go mod tidy\n echo \"Running tests with coverage...\"\n go test -mod=readonly -timeout=30m -v -cover -race -coverprofile=coverage.tmp ./pkg/...\n cat coverage.tmp | grep -v '\\.pb\\.go:\\|_mock\\.go:' > coverage.out\n python ./test-tools/find_uncover_err.py coverage.out filtered_cover_no_err_branch.out\n gcov2lcov -infile=filtered_cover_no_err_branch.out -outfile=coverage.lcov\n\n run_cpp_test:\n description: \"Run cpp cov tests and collect artifacts\"\n parameters:\n extra_bazel_args:\n type: string\n default: \"\"\n find_executable_flag:\n type: string\n default: \"-executable\"\n steps:\n - run:\n name: \"Cpp Test\"\n command: |\n set +e\n declare -i test_status\n\n echo \"Running tests with coverage...\"\n bazelisk --host_jvm_args=-Xmx8g coverage //engine/... \\\n << parameters.extra_bazel_args >> \\\n --combined_report=lcov \\\n --jobs=auto \\\n --ui_event_filters=-info,-debug,-warning \\\n --test_output=errors | tee test_result.log\n\n # Capture the exit status of the Bazel command\n test_status=${PIPESTATUS[0]}\n if [ ${test_status} -eq 0 ]; then\n echo \"Processing coverage...\"\n lcov --remove bazel-out/_coverage/_coverage_report.dat '*.pb.h' '*.pb.cc' -o bazel-out/_coverage/_coverage_report_filtered.dat\n else\n echo \"Bazel coverage failed, skipping lcov processing. Archiving binaries and logs...\"\n find bazel-bin/ << parameters.find_executable_flag >> -type f -name \"*_test\" -print0 | xargs -0 tar -cvzf test_binary.tar.gz\n find bazel-testlogs/ -type f -name \"test.log\" -print0 | xargs -0 tar -cvzf test_logs.tar.gz\n fi\n\n sh ../devtools/rename-junit-xml.sh\n exit ${test_status}\n\n # ref: https://support.circleci.com/hc/en-us/articles/14114124583195-How-to-set-a-custom-maximum-job-duration\n cancel_after_timeout:\n description: \"Cancel job if it takes too long\"\n parameters:\n timeout:\n type: string\n default: \"60m\"\n steps:\n - run:\n name: Set maximum job duration to << parameters.timeout >>\n background: true\n command: |\n sleep << parameters.timeout >>\n curl --request POST \\\n --url https://circleci.com/api/v2/project/gh/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/job/$CIRCLE_BUILD_NUM/cancel \\\n --header \"Circle-Token: $CIRCLECI_API_TOKEN\"\n\njobs:\n linux_diff_cov:\n docker:\n - image: secretflow/scql-ci:latest\n resource_class: \"2xlarge\"\n steps:\n - cancel_after_timeout:\n timeout: \"90m\"\n - checkout\n - run:\n name: \"Install Dependencies (lcov, diff-cover)\"\n command: |\n apt-get update && apt-get install -y lcov\n pip install diff-cover\n - setup_go_test\n - setup_cpp_test\n - run_go_test\n - run_cpp_test:\n extra_bazel_args: \"-c opt\"\n - run:\n name: \"Debug Coverage Files\"\n command: |\n echo \"=== Debug Information ===\"\n echo \"Current working directory: $(pwd)\"\n echo \"Available coverage files:\"\n find . -name \"*.lcov\" -o -name \"*coverage*\" -type f | head -10\n\n if [ -f \"coverage.lcov\" ]; then\n echo \"Go coverage file size: $(wc -l < coverage.lcov) lines\"\n echo \"Go coverage sample:\"\n head -10 coverage.lcov\n fi\n\n if [ -f \"bazel-out/_coverage/_coverage_report_filtered.dat\" ]; then\n echo \"C++ coverage file size: $(wc -l < bazel-out/_coverage/_coverage_report_filtered.dat) lines\"\n echo \"C++ coverage sample:\"\n head -10 bazel-out/_coverage/_coverage_report_filtered.dat\n fi\n\n echo \"Git status:\"\n git status --porcelain\n echo \"Recent commits:\"\n git log --oneline -5\n - run:\n name: \"Calculate and Report Incremental Coverage\"\n command: |\n echo \"Calculating incremental coverage between << pipeline.parameters.base_commit >> and ${CIRCLE_SHA1}\"\n git config remote.origin.url \"https://github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}.git\"\n git config advice.objectNameWarning false\n\n if ! git rev-parse --verify << pipeline.parameters.base_commit >> >/dev/null 2>&1; then\n echo \"Base commit not found locally, fetching...\"\n git fetch origin << pipeline.parameters.base_commit >> || \\\n git fetch origin --depth=100 || \\\n git fetch origin\n fi\n\n if ! git rev-parse --verify << pipeline.parameters.base_commit >> >/dev/null 2>&1; then\n echo \"Error: Base commit << pipeline.parameters.base_commit >> not found\"\n exit 1\n fi\n if ! git rev-parse --verify ${CIRCLE_SHA1} >/dev/null 2>&1; then\n echo \"Error: Current commit ${CIRCLE_SHA1} not found\"\n exit 1\n fi\n echo \"Base commit: $(git rev-parse << pipeline.parameters.base_commit >>)\"\n echo \"Head commit: $(git rev-parse ${CIRCLE_SHA1})\"\n\n # Create a directory to store the reports\n mkdir -p coverage-reports\n FINAL_EXIT_CODE=0\n\n # --- 1. Go Incremental Coverage ---\n echo \"\"\n echo \"--- Go Incremental Coverage ---\"\n if [ -f \"coverage.lcov\" ]; then\n git diff << pipeline.parameters.base_commit >>..${CIRCLE_SHA1} -- '*.go' > go_diff.txt\n diff-cover coverage.lcov \\\n --diff-file=go_diff.txt \\\n --fail-under=80 \\\n --exclude='*_test.go' \\\n --exclude='*.pb.go' \\\n --exclude='*_mock.go' \\\n --format html:coverage-reports/go_diff_coverage.html \\\n || { echo \"Go incremental coverage check FAILED.\"; FINAL_EXIT_CODE=1; }\n else\n echo \"Go coverage file (coverage.lcov) not found. Skipping.\"\n fi\n\n # --- 2. C++ Incremental Coverage ---\n echo \"\"\n echo \"--- C++ Incremental Coverage ---\"\n CPP_COV_FILE=\"bazel-out/_coverage/_coverage_report_filtered.dat\"\n if [ -f \"${CPP_COV_FILE}\" ]; then\n git diff << pipeline.parameters.base_commit >>..${CIRCLE_SHA1} -- '*.cpp' '*.h' '*.cc' > cpp_diff.txt\n diff-cover ${CPP_COV_FILE} \\\n --diff-file=cpp_diff.txt \\\n --fail-under=80 \\\n --exclude='*.pb.h' \\\n --exclude='*.pb.cc' \\\n --format html:coverage-reports/cpp_diff_coverage.html \\\n || { echo \"C++ incremental coverage check FAILED.\"; FINAL_EXIT_CODE=1; }\n else\n echo \"C++ coverage file (${CPP_COV_FILE}) not found. Skipping.\"\n fi\n\n exit ${FINAL_EXIT_CODE}\n - store_artifacts:\n path: coverage-reports\n - store_artifacts:\n path: coverage.lcov\n destination: go-coverage.lcov\n - store_artifacts:\n path: bazel-out/_coverage/_coverage_report_filtered.dat\n destination: cpp-coverage.lcov\n\nworkflows:\n run_diff_cov_workflow:\n jobs:\n - linux_diff_cov\n" }, { "path": ".circleci/full-unittest-config.yml", "content": "# Copyright 2023 Ant Group Co., Ltd.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# Use the latest 2.1 version of CircleCI pipeline process engine.\n# See: https://circleci.com/docs/2.0/configuration-reference\n\nversion: 2.1\n\ncommands:\n setup_cpp_test:\n steps:\n - run:\n name: \"Checkout devtools\"\n command: git clone --depth=1 https://github.com/secretflow/devtools.git ../devtools\n - run:\n name: Setup BuildBuddy Cache\n command: ../devtools/bazel_cache_setup.py\n\n run_go_test:\n description: \"Run go tests\"\n steps:\n - run:\n name: \"Go Test\"\n command: |\n set +e\n go mod tidy\n echo \"Running unit tests...\"\n go test -mod=readonly -timeout=30m -v -short ./pkg/...\n\n run_cpp_test:\n description: \"Run cpp tests and collect artifacts on fail\"\n parameters:\n extra_bazel_args:\n type: string\n default: \"\"\n find_executable_flag:\n type: string\n default: \"-executable\"\n steps:\n - run:\n name: \"Cpp Test\"\n command: |\n set +e\n declare -i test_status\n\n echo \"Running unit tests...\"\n bazelisk --host_jvm_args=-Xmx8g test //engine/... \\\n << parameters.extra_bazel_args >> \\\n --jobs=auto \\\n --ui_event_filters=-info,-debug,-warning \\\n --test_output=errors | tee test_result.log\n\n # Capture the exit status of the Bazel command\n test_status=${PIPESTATUS[0]}\n\n sh ../devtools/rename-junit-xml.sh\n if [ ${test_status} -ne 0 ]; then\n echo \"Tests failed. Archiving binaries and logs...\"\n find bazel-bin/ << parameters.find_executable_flag >> -type f -name \"*_test\" -print0 | xargs -0 tar -cvzf test_binary.tar.gz\n find bazel-testlogs/ -type f -name \"test.log\" -print0 | xargs -0 tar -cvzf test_logs.tar.gz\n fi\n\n exit ${test_status}\n\n # ref: https://support.circleci.com/hc/en-us/articles/14114124583195-How-to-set-a-custom-maximum-job-duration\n cancel_after_timeout:\n description: \"Cancel job if it takes too long\"\n parameters:\n timeout:\n type: string\n default: \"60m\"\n steps:\n - run:\n name: Set maximum job duration to << parameters.timeout >>\n background: true\n command: |\n sleep << parameters.timeout >>\n curl --request POST \\\n --url https://circleci.com/api/v2/project/gh/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/job/$CIRCLE_BUILD_NUM/cancel \\\n --header \"Circle-Token: $CIRCLECI_API_TOKEN\"\n\njobs:\n linux_go_ut:\n docker:\n - image: secretflow/scql-ci:latest\n parameters:\n resource_class:\n type: string\n resource_class: << parameters.resource_class >>\n steps:\n - cancel_after_timeout:\n timeout: \"20m\"\n - checkout\n - run_go_test\n\n linux_cpp_ut:\n docker:\n - image: secretflow/scql-ci:latest\n parameters:\n resource_class:\n type: string\n resource_class: << parameters.resource_class >>\n steps:\n - cancel_after_timeout:\n timeout: \"60m\"\n - checkout\n - setup_cpp_test\n - run_cpp_test:\n extra_bazel_args: \"-c opt\"\n find_executable_flag: \"-executable\"\n - store_test_results:\n path: test-results\n - store_artifacts:\n when: on_fail\n path: test_binary.tar.gz\n - store_artifacts:\n when: on_fail\n path: test_logs.tar.gz\n\n macOS_go_ut:\n macos:\n xcode: 16.3.0\n resource_class: m4pro.medium\n steps:\n - cancel_after_timeout:\n timeout: \"20m\"\n - checkout\n - run:\n name: \"Install homebrew dependencies\"\n command: |\n brew install wget go\n - run_go_test\n\n macOS_cpp_ut:\n macos:\n xcode: 16.3.0\n resource_class: m4pro.medium\n steps:\n - cancel_after_timeout:\n timeout: \"60m\"\n - checkout\n - run:\n name: \"Install homebrew dependencies\"\n command: |\n brew install bazelisk cmake ninja libomp wget go@1.24 md5sha1sum\n brew link go@1.24\n - setup_cpp_test\n - run_cpp_test:\n extra_bazel_args: \"\"\n find_executable_flag: \"-perm +111\"\n - store_test_results:\n path: test-results\n - store_artifacts:\n when: on_fail\n path: test_binary.tar.gz\n - store_artifacts:\n when: on_fail\n path: test_logs.tar.gz\n\nworkflows:\n run_go_ut:\n jobs:\n - linux_go_ut:\n matrix:\n parameters:\n resource_class: [\"large\", \"arm.large\"]\n - macOS_go_ut\n run_cpp_ut:\n jobs:\n - linux_cpp_ut:\n matrix:\n parameters:\n resource_class: [\"2xlarge\", \"arm.2xlarge\"]\n - macOS_cpp_ut\n" }, { "path": ".circleci/lite-unittest-config.yml", "content": "# Copyright 2025 Ant Group Co., Ltd.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# Use the latest 2.1 version of CircleCI pipeline process engine.\n# See: https://circleci.com/docs/2.0/configuration-reference\n\nversion: 2.1\n\nparameters:\n run_go_ut:\n type: boolean\n default: false\n run_cpp_ut:\n type: boolean\n default: false\n\ncommands:\n setup_cpp_test:\n steps:\n - run:\n name: \"Checkout devtools\"\n command: git clone --depth=1 https://github.com/secretflow/devtools.git ../devtools\n - run:\n name: Setup BuildBuddy Cache\n command: ../devtools/bazel_cache_setup.py\n\n run_go_test:\n description: \"Run go tests\"\n steps:\n - run:\n name: \"Go Test\"\n command: |\n set +e\n go mod tidy\n echo \"Running unit tests...\"\n go test -mod=readonly -timeout=30m -v -short ./pkg/...\n\n run_cpp_test:\n description: \"Run cpp tests and collect artifacts on fail\"\n steps:\n - run:\n name: \"Cpp Test\"\n command: |\n set +e\n declare -i test_status\n\n echo \"Running unit tests...\"\n bazelisk --host_jvm_args=-Xmx8g test //engine/... \\\n -c opt \\\n --jobs=16 \\\n --local_ram_resources=20480 \\\n --ui_event_filters=-info,-debug,-warning \\\n --test_output=errors | tee test_result.log\n\n # Capture the exit status of the Bazel command\n test_status=${PIPESTATUS[0]}\n\n sh ../devtools/rename-junit-xml.sh\n if [ ${test_status} -ne 0 ]; then\n echo \"Tests failed. Archiving binaries and logs...\"\n find bazel-bin/ -executable -type f -name \"*_test\" -print0 | xargs -0 tar -cvzf test_binary.tar.gz\n find bazel-testlogs/ -type f -name \"test.log\" -print0 | xargs -0 tar -cvzf test_logs.tar.gz\n fi\n\n exit ${test_status}\n\n # ref: https://support.circleci.com/hc/en-us/articles/14114124583195-How-to-set-a-custom-maximum-job-duration\n cancel_after_timeout:\n description: \"Cancel job if it takes too long\"\n parameters:\n timeout:\n type: string\n default: \"60m\"\n steps:\n - run:\n name: Set maximum job duration to << parameters.timeout >>\n background: true\n command: |\n sleep << parameters.timeout >>\n curl --request POST \\\n --url https://circleci.com/api/v2/project/gh/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/job/$CIRCLE_BUILD_NUM/cancel \\\n --header \"Circle-Token: $CIRCLECI_API_TOKEN\"\n\njobs:\n linux_go_ut:\n docker:\n - image: secretflow/scql-ci:latest\n resource_class: \"large\"\n steps:\n - cancel_after_timeout:\n timeout: \"20m\"\n - checkout\n - run_go_test\n\n linux_cpp_ut:\n docker:\n - image: secretflow/scql-ci:latest\n resource_class: \"2xlarge\"\n steps:\n - cancel_after_timeout:\n timeout: \"60m\"\n - checkout\n - setup_cpp_test\n - run_cpp_test\n - store_test_results:\n path: test-results\n - store_artifacts:\n when: on_fail\n path: test_binary.tar.gz\n - store_artifacts:\n when: on_fail\n path: test_logs.tar.gz\n\nworkflows:\n run_go_ut:\n when: << pipeline.parameters.run_go_ut >>\n jobs:\n - linux_go_ut\n\n run_cpp_ut:\n when: << pipeline.parameters.run_cpp_ut >>\n jobs:\n - linux_cpp_ut\n" }, { "path": ".circleci/path-filtering/unittest.conf", "content": "api/.* run_go_ut true\ncmd/.* run_go_ut true\npkg/.* run_go_ut true\n.circleci/lite-unittest-config.yml run_go_ut true\nbazel/.* run_cpp_ut true\nengine/.* run_cpp_ut true\n.bazelrc run_cpp_ut true\n.bazeliskrc run_cpp_ut true\nMODULE.bazel run_cpp_ut true\n.circleci/lite-unittest-config.yml run_cpp_ut true\n" }, { "path": ".clang-format", "content": "# Use the Google style in this project.\nBasedOnStyle: Google\n\nIncludeBlocks: Regroup\nIncludeCategories:\n - Regex: '^<.*\\.h>'\n Priority: 1\n - Regex: \"^<.*\"\n Priority: 2\n - Regex: '.*\\.pb\\.h\"$'\n Priority: 5\n - Regex: '^\"engine.*'\n Priority: 4\n - Regex: '^\".*'\n Priority: 3\n" }, { "path": ".clang-tidy", "content": "Checks: \"abseil-cleanup-ctad,\n abseil-faster-strsplit-delimiter,\n abseil-duration-*,\n abseil-no-namespace,\n abseil-redundant-strcat-calls,\n abseil-str-cat-append,\n abseil-string-find-startswith,\n abseil-upgrade-duration-conversions\n bugprone-*,\n -bugprone-easily-swappable-parameters,\n -bugprone-implicit-widening-of-multiplication-result,\n -bugprone-narrowing-conversions,\n google-build-using-namespace,\n google-explicit-constructor,\n google-global-names-in-headers,\n google-readability-casting,\n google-runtime-int,\n google-runtime-operator,\n misc-unused-using-decls,\n modernize-*,\n -modernize-use-trailing-return-type,\n -modernize-avoid-c-arrays,\n -modernize-return-braced-init-list,\n -modernize-use-nodiscard,\n performance-*,\n readability-*,\n -readability-else-after-return,\n -readability-identifier-length,\n -readability-function-cognitive-complexity,\n -readability-magic-numbers,\n -readability-named-parameter,\n -readability-math-missing-parentheses,\n -readability-redundant-access-specifiers,\n -readability-simplify-boolean-expr,\n concurrency-mt-unsafe\"\n\nCheckOptions:\n - key: bugprone-argument-comment.StrictMode\n value: 1\n\n - key: bugprone-dangling-handle.HandleClasses\n value: \"std::basic_string_view;std::experimental::basic_string_view;absl::string_view\"\n\n - key: misc-non-private-member-variables-in-classes.IgnoreClassesWithAllMemberVariablesBeingPublic\n value: 1\n\n # Ignore GoogleTest function macros.\n - key: readability-identifier-naming.FunctionIgnoredRegexp\n value: \"(TEST|TEST_F|TEST_P|INSTANTIATE_TEST_SUITE_P|MOCK_METHOD|TYPED_TEST)\"\n\n - key: readability-identifier-naming.ClassCase\n value: \"CamelCase\"\n\n - key: readability-identifier-naming.EnumCase\n value: \"CamelCase\"\n\n - key: readability-identifier-naming.EnumConstantCase\n value: \"CamelCase\"\n\n - key: readability-identifier-naming.ParameterCase\n value: \"lower_case\"\n\n - key: readability-identifier-naming.PrivateMemberCase\n value: \"lower_case\"\n\n - key: readability-identifier-naming.PrivateMemberSuffix\n value: \"_\"\n\n - key: readability-identifier-naming.StructCase\n value: \"CamelCase\"\n\n - key: readability-identifier-naming.TypeAliasCase\n value: \"CamelCase\"\n\n - key: readability-identifier-naming.UnionCase\n value: \"CamelCase\"\n\n - key: readability-identifier-naming.FunctionCase\n value: \"CamelBack\"\n" }, { "path": ".coveralls.yml", "content": "# Configure whether the Coveralls status check fails when coverage decreases\nfail_on_coverage_decrease: false\n\ncoverage:\n status:\n project:\n go-tests:\n # Set a coverage threshold for the changed lines of code.\n target: 70%\n\n cpp-tests:\n target: 70%\n\n patch:\n default:\n target: 70%\n\ncomment:\n pull_request:\n # update a comment\n behavior: default\n\n # diff: Shows overall coverage change and coverage of changed lines\n # flags: If upload multiple reports (C++ and Go), they will be displayed separately\n # files: Lists the files where coverage has changed\n layout: \"diff, flags, files\"" }, { "path": ".devcontainer/Dockerfile", "content": "FROM secretflow/ubuntu-base-ci:latest\n\nARG TARGETPLATFORM\nARG GO_VERSION=1.24.0\n\n# install go\nRUN if [ \"$TARGETPLATFORM\" = \"linux/arm64\" ] ; \\\n then \\\n GO_ARCH=arm64 && \\\n GO_SHA256SUM=c3fa6d16ffa261091a5617145553c71d21435ce547e44cc6dfb7470865527cc7 ; \\\n else \\\n GO_ARCH=amd64 && \\\n GO_SHA256SUM=dea9ca38a0b852a74e81c26134671af7c0fbe65d81b0dc1c5bfe22cf7d4c8858 ; \\\n fi \\\n && url=\"https://golang.google.cn/dl/go${GO_VERSION}.linux-${GO_ARCH}.tar.gz\"; \\\n wget --no-check-certificate -O go.tgz \"$url\"; \\\n echo \"${GO_SHA256SUM} *go.tgz\" | sha256sum -c -; \\\n tar -C /usr/local -xzf go.tgz; \\\n rm go.tgz;\n\nENV GOPATH=\"/usr/local\"\nENV PATH=\"/usr/local/go/bin:${GOPATH}/bin:${PATH}\"\n\nRUN apt update \\\n && apt upgrade -y \\\n && apt install -y protobuf-compiler \\\n && apt clean\n\n\n# Create a non-root user 'vscode' with sudo privileges\n# The user ID 1000 is common for the primary user in Linux distributions.\n# The GID 1000 corresponds to the user's primary group.\nRUN groupadd --gid 1000 vscode && \\\n useradd --uid 1000 --gid 1000 --shell /bin/bash --create-home vscode && \\\n echo \"vscode ALL=(ALL) NOPASSWD:ALL\" >> /etc/sudoers\n\n# Set the default user for subsequent commands\nUSER vscode\n\n# Set the working directory inside the container\nWORKDIR /workspaces/scql\n\n# Set default shell to bash for the vscode user\nENV SHELL /bin/bash\n\n# You can add more SCQL-specific dependencies here if needed,\n# although 'make install-dev-deps' in postCreateCommand should handle most project dependencies.\n\n# Keep the container running (optional, useful for debugging setup)\n# CMD [\"sleep\", \"infinity\"]\n" }, { "path": ".devcontainer/devcontainer.json", "content": "// For format details, see https://aka.ms/devcontainer.json. For config options, see the\n// README at: https://github.com/devcontainers/templates/tree/main/src/cpp\n{\n // A name for the dev container which can be displayed in UI.\n \"name\": \"SCQL Dev Container\",\n\n // Sets the run context to one level up instead of the .devcontainer folder.\n \"context\": \"..\",\n\n // Update the VARIANT arg in docker-compose.yml to pick a Debian OS version: bullseye, buster\n // Use Python 3.10 to align with SCQL's CI/CD environment if possible, adjust if needed.\n \"build\": {\n \"dockerfile\": \"Dockerfile\"\n },\n\n // Features to add to the dev container. More info: https://containers.dev/features.\n // \"features\": {},\n\n // Use 'forwardPorts' to make a list of ports inside the container available locally.\n // \"forwardPorts\": [],\n\n // Use 'postCreateCommand' to run commands after the container is created.\n // Installs development dependencies using the Makefile target.\n \"postCreateCommand\": \"echo 'hello world'\",\n\n // Configure tool-specific properties.\n // \"customizations\": {},\n\n // Specifies the user the container will run as. Default is root.\n // Using a non-root user 'vscode' is recommended for security.\n \"remoteUser\": \"vscode\",\n\n // Mount the workspace folder.\n \"workspaceFolder\": \"/workspaces/scql\",\n \"workspaceMount\": \"source=${localWorkspaceFolder},target=/workspaces/scql,type=bind,consistency=cached\"\n}\n" }, { "path": ".github/CODEOWNERS", "content": "# default reviewers for everything in this repo.\n* @secretflow/scql-dev\n" }, { "path": ".github/ISSUE_TEMPLATE/Consulting_issue_template.yaml", "content": "name: Consulting Template\ndescription: Ask SCQL related questions\nbody:\n - type: markdown\n attributes:\n value: |\n Please ensure that you are reporting the consultation issue on GitHub.（CCL、Perfomance、Feature、Documentation、Others）\n - type: dropdown\n id: issue-type\n attributes:\n label: Issue Type\n description: What type of issue would you like to report?\n multiple: false\n options:\n - CCL\n - Perfomance\n - Feature\n - Documentation\n - Others\n validations:\n required: true\n - type: dropdown\n id: searched-for-existing-issues\n attributes:\n label: Have you searched for existing issues? \n description: It is recommended to search existing [documentation](https://www.secretflow.org.cn/zh-CN/docs/scql/main/topics) and [issues](https://github.com/secretflow/scql/issues) first\n options:\n - 'Yes'\n - 'No'\n validations:\n required: true\n - type: input\n id: link\n attributes:\n label: Link to Relevant Documentation\n description: For faster problem-solving, if there are relevant documents, please attach links.\n placeholder: e.g., https://www.secretflow.org.cn/zh-CN/docs/scql/main/topics/ccl/intro\n validations:\n required: false\n - type: textarea\n id: Question-Details\n attributes:\n label: Question Details\n description: Please detail your issue with observed versus expected behavior and attempted solutions to expedite resolution.\n placeholder: Describe the questions you want to consult and what you want to do\n value:\n render: shell\n validations:\n required: true\n" }, { "path": ".github/ISSUE_TEMPLATE/Error_Issue_Template.yaml", "content": "name: Error Template\ndescription: Thank you for reporting the issue!\nbody:\n - type: markdown\n attributes:\n value: |\n Please ensure that you are reporting the consultation issue on GitHub.（Install/Build、Running、CCL、Others）\n - type: dropdown\n id: issue-type\n attributes:\n label: Issue Type\n description: What type of issue would you like to report?\n multiple: false\n options:\n - Install/Build\n - Running\n - CCL\n - Others\n validations:\n required: true\n - type: dropdown\n id: searched-for-existing-issues\n attributes:\n label: Have you searched for existing issues? \n description: It is recommended to search existing [documentation](https://www.secretflow.org.cn/zh-CN/docs/scql/main/topics) and [issues](https://github.com/secretflow/scql/issues) first\n options:\n - 'Yes'\n - 'No'\n validations:\n required: true\n - type: input\n id: OS\n attributes:\n label: OS Platform and Distribution\n description:\n placeholder: e.g., Linux Ubuntu 18.04\n validations:\n required: true\n - type: input\n id: scql-version\n attributes:\n label: SCQL Version\n description:\n placeholder: e.g., SCQL 0.7.0b0\n validations:\n required: true\n - type: textarea\n id: what-happened\n attributes:\n label: What happend and What you expected to happen.\n description: A clear and concise description of what the bug is.\n placeholder: Describe the bug, expected behavior.\n value:\n render: shell\n validations:\n required: true\n - type: textarea\n id: scql-config\n attributes:\n label: Configuration used to run SCQL.\n description: Supply SCQL runtime config (.yaml, .conf) and, for CCL issues, supply authorization info and SQL statements.\n placeholder: |\n # - For install/deploy, provide files ending with `.yaml` and `.conf`.\n # - For CCL, provide the authorization details and CCL SQL.\n value:\n render: shell\n validations:\n required: true\n - type: textarea\n id: log-output\n attributes:\n label: SCQL log output.\n description: Supply relevant log output (docker logs -f xxx-broker-xxx/xxx-engine-xxx); For multi-party (e.g. Alice, Bob) scenarios, include all corresponding logs.\n placeholder: |\n # alice.log\n .......\n # bob.log\n .......\n value:\n render: shell\n validations:\n required: true\n" }, { "path": ".github/workflows/black.yml", "content": "---\nname: Python Linter\non:\n push:\n branches:\n - main\n paths:\n - '**.py'\n pull_request:\n branches:\n - main\n paths:\n - '**.py'\npermissions:\n contents: read\njobs:\n python-linter:\n uses: secretflow/.github/.github/workflows/python-linter.yml@main" }, { "path": ".github/workflows/buildifier.yml", "content": "---\nname: Bazel files linter\non:\n push:\n branches:\n - main\n pull_request:\n branches:\n - main\npermissions:\n contents: read\njobs:\n bazel-formatting-check:\n uses: secretflow/.github/.github/workflows/bazel-linter.yml@main" }, { "path": ".github/workflows/cla.yml", "content": "---\nname: CLA Assistant\non:\n issue_comment:\n types: [created]\n pull_request_target:\n types: [opened, closed, synchronize]\njobs:\n CLAssistant:\n uses: secretflow/.github/.github/workflows/cla.yml@main\n secrets: inherit\n" }, { "path": ".github/workflows/clang-format-linter.yml", "content": "---\nname: Run clang-format Linter\non:\n push:\n branches:\n - main\n paths:\n - '**.cc'\n - '**.cpp'\n - '**.hpp'\n - '**.h'\n - '**.proto'\n pull_request:\n branches:\n - main\n paths:\n - '**.cc'\n - '**.cpp'\n - '**.hpp'\n - '**.h'\n - '**.proto'\npermissions:\n contents: read\njobs:\n run-clang-format:\n uses: secretflow/.github/.github/workflows/clang-format.yml@main" }, { "path": ".github/workflows/codeql.yml", "content": "# For most projects, this workflow file will not need changing; you simply need\n# to commit it to your repository.\n#\n# You may wish to alter this file to override the set of languages analyzed,\n# or to provide custom queries or build logic.\n#\n# ******** NOTE ********\n# We have attempted to detect the languages in your repository. Please check\n# the `language` matrix defined below to confirm you have the correct set of\n# supported CodeQL languages.\n#\nname: \"CodeQL\"\n\non:\n push:\n branches: [\"main\"]\n pull_request:\n # The branches below must be a subset of the branches above\n branches: [\"main\"]\n schedule:\n - cron: \"0 0 * * 1\"\n\npermissions:\n contents: read\n\njobs:\n analyze:\n name: Analyze\n runs-on: ubuntu-latest\n permissions:\n actions: read\n contents: read\n security-events: write\n\n strategy:\n fail-fast: false\n matrix:\n language: [\"cpp\", \"go\", \"python\"]\n # CodeQL supports [ $supported-codeql-languages ]\n # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support\n\n steps:\n - name: Harden the runner (Audit all outbound calls)\n uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0\n with:\n egress-policy: audit\n\n - name: Checkout repository\n uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2\n\n # Initializes the CodeQL tools for scanning.\n - name: Initialize CodeQL\n uses: github/codeql-action/init@28deaeda66b76a05916b6923827895f2b14ab387 # v3.28.16\n with:\n languages: ${{ matrix.language }}\n # If you wish to specify custom queries, you can do so here or in a config file.\n # By default, queries listed here will override any specified in a config file.\n # Prefix the list here with \"+\" to use these queries and those in the config file.\n\n # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).\n # If this step fails, then you should remove it and run the build manually (see below)\n - name: Autobuild\n uses: github/codeql-action/autobuild@28deaeda66b76a05916b6923827895f2b14ab387 # v3.28.16\n\n # ℹ️ Command-line programs to run using the OS shell.\n # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun\n\n # If the Autobuild fails above, remove it and uncomment the following three lines.\n # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.\n\n # - run: |\n # echo \"Run, Build Application using script\"\n # ./location_of_script_within_repo/buildscript.sh\n\n - name: Perform CodeQL Analysis\n uses: github/codeql-action/analyze@28deaeda66b76a05916b6923827895f2b14ab387 # v3.28.16\n with:\n category: \"/language:${{matrix.language}}\"\n" }, { "path": ".github/workflows/dependency-review.yml", "content": "# Dependency Review Action\n#\n# This Action will scan dependency manifest files that change as part of a Pull Request,\n# surfacing known-vulnerable versions of the packages declared or updated in the PR.\n# Once installed, if the workflow run is marked as required,\n# PRs introducing known-vulnerable packages will be blocked from merging.\n#\n# Source repository: https://github.com/actions/dependency-review-action\nname: 'Dependency Review'\non: [pull_request]\n\npermissions:\n contents: read\n\njobs:\n dependency-review:\n runs-on: ubuntu-latest\n steps:\n - name: Harden the runner (Audit all outbound calls)\n uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0\n with:\n egress-policy: audit\n\n - name: 'Checkout Repository'\n uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2\n - name: 'Dependency Review'\n uses: actions/dependency-review-action@67d4f4bd7a9b17a0db54d2a7519187c65e339de8 # v4\n" }, { "path": ".github/workflows/docs-check.yml", "content": "---\nname: Check Docs\non:\n push:\n branches:\n - main\n paths:\n - 'docs/**'\n pull_request:\n branches:\n - main\n paths:\n - 'docs/**'\n\npermissions:\n contents: read\n pull-requests: read\n\njobs:\n check-docs:\n name: check docs\n runs-on: [ubuntu-latest]\n container:\n image: secretflow/scql-ci:20250228\n steps:\n - name: Harden the runner (Audit all outbound calls)\n uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0\n with:\n egress-policy: audit\n\n - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2\n with:\n fetch-depth: '0'\n - name: make docs\n run: |\n # avoid {{ and {# in docs\n if [ $(grep -E '({{|{#)' --include=\\*.{rst,md,ipynb,po} --exclude=run-scql-on-kuscia.* -rnw 'docs' | wc -l) != 0 ];\n then\n echo \"({{|{#) is not allowed in rst,md,ipynb,po files. Thank you for cooperation.\"\n grep -E '({{|{#)' --include=\\*.{rst,md,ipynb,po} --exclude=run-scql-on-kuscia.* -rnw 'docs'\n exit 1\n fi" }, { "path": ".github/workflows/docs-publish.yml", "content": "# Copyright 2025 Ant Group Co., Ltd.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nname: Publish Docs\non:\n push:\n branches:\n - main\n tags:\n - \"*\"\n\njobs:\n check-docs:\n name: check docs\n runs-on: [ubuntu-latest]\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-node@v4\n with:\n node-version: \"22\"\n - uses: actions/setup-python@v5\n with:\n python-version: \"3.10\"\n - name: pubilsh docs\n run: |\n python3 -m venv ~/.venv/docs\n source ~/.venv/docs/bin/activate\n python -m pip install -r docs/requirements.txt\n secretflow-doctools build --lang zh_CN --lang en\n secretflow-doctools publish \\\n --name @secretflow/x-scql \\\n --index-js docs/_build/esm/index.js\n env:\n DRY_RUN: \"0\" # omit in test runs\n DOCTOOLS_PUBLISH_NPM_TOKEN: ${{secrets.DOCTOOLS_PUBLISH_NPM_TOKEN}}\n" }, { "path": ".github/workflows/golangci-lint.yml", "content": "---\nname: Golangci-Lint\n\non:\n push:\n branches:\n - main\n paths:\n - '**.go'\n - '.golangci-lint.yml'\n pull_request:\n branches:\n - main\n paths:\n - '**.go'\n - '.golangci-lint.yml'\n\nenv:\n GO_VERSION: 1.24\n GOLANGCI_LINT_VERSION: v2.0\n\npermissions:\n # Required: allow read access to the content for analysis.\n contents: read\n # Optional: allow read access to pull request. Use with `only-new-issues` option.\n pull-requests: read\n\njobs:\n detect-modules:\n runs-on: ubuntu-latest\n outputs:\n modules: ${{ steps.set-modules.outputs.modules }}\n steps:\n - name: Harden the runner (Audit all outbound calls)\n uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0\n with:\n egress-policy: audit\n\n - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2\n - uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0\n with:\n go-version: ${{ env.GO_VERSION }}\n - id: set-modules\n run: echo \"modules=$(go list -m -json | jq -s '.' | jq -c '[.[].Dir]')\" >> $GITHUB_OUTPUT\n\n golangci-lint:\n needs: detect-modules\n runs-on: ubuntu-latest\n strategy:\n matrix:\n modules: ${{ fromJSON(needs.detect-modules.outputs.modules) }}\n steps:\n - name: Harden the runner (Audit all outbound calls)\n uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0\n with:\n egress-policy: audit\n\n - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2\n - uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0\n with:\n go-version: ${{ env.GO_VERSION }}\n - name: golangci-lint ${{ matrix.modules }}\n uses: golangci/golangci-lint-action@1481404843c368bc19ca9406f87d6e0fc97bdcfd # v7.0.0\n with:\n version: ${{ env.GOLANGCI_LINT_VERSION }}\n working-directory: ${{ matrix.modules }}" }, { "path": ".github/workflows/govulncheck.yml", "content": "---\nname: Go Vulnerability Check\n\non:\n push:\n branches:\n - main\n paths:\n - '**.go'\n - 'go.mod'\n - 'go.sum'\n - '.github/workflows/govulncheck.yml'\n pull_request:\n branches:\n - main\n paths:\n - '**.go'\n - 'go.mod'\n - 'go.sum'\n - '.github/workflows/govulncheck.yml'\n\njobs:\n govulncheck_job:\n runs-on: ubuntu-latest\n name: Run govulncheck\n steps:\n - id: govulncheck\n uses: golang/govulncheck-action@v1\n with:\n go-package: ./...\n go-version-file: 'go.mod'\n output-format: 'text'" }, { "path": ".github/workflows/license-check.yml", "content": "---\nname: License Check\non:\n push:\n branches:\n - main\n pull_request:\n branches:\n - main\npermissions:\n contents: read\njobs:\n license-checker:\n uses: secretflow/.github/.github/workflows/license-check.yml@main" }, { "path": ".github/workflows/oscp.yml", "content": "name: Unassign Stale OSCP Issues\n\non:\n schedule:\n - cron: \"0 */6 * * *\" # Every 6 hours\n\njobs:\n unassign-stale-issues:\n uses: secretflow/.github/.github/workflows/oscp-unassign.yml@main\n" }, { "path": ".github/workflows/scorecards.yml", "content": "# This workflow uses actions that are not certified by GitHub. They are provided\n# by a third-party and are governed by separate terms of service, privacy\n# policy, and support documentation.\n\nname: Scorecard supply-chain security\non:\n # For Branch-Protection check. Only the default branch is supported. See\n # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection\n branch_protection_rule:\n # To guarantee Maintained check is occasionally updated. See\n # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained\n schedule:\n - cron: '20 7 * * 2'\n push:\n branches: [\"main\"]\n\n# Declare default permissions as read only.\npermissions: read-all\n\njobs:\n analysis:\n name: Scorecard analysis\n runs-on: ubuntu-latest\n permissions:\n # Needed to upload the results to code-scanning dashboard.\n security-events: write\n # Needed to publish results and get a badge (see publish_results below).\n id-token: write\n contents: read\n actions: read\n # To allow GraphQL ListCommits to work\n issues: read\n pull-requests: read\n # To detect SAST tools\n checks: read\n\n steps:\n - name: Harden the runner (Audit all outbound calls)\n uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0\n with:\n egress-policy: audit\n\n - name: \"Checkout code\"\n uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2\n with:\n persist-credentials: false\n\n - name: \"Run analysis\"\n uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # v2.4.0\n with:\n results_file: results.sarif\n results_format: sarif\n # (Optional) \"write\" PAT token. Uncomment the `repo_token` line below if:\n # - you want to enable the Branch-Protection check on a *public* repository, or\n # - you are installing Scorecards on a *private* repository\n # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat.\n # repo_token: ${{ secrets.SCORECARD_TOKEN }}\n\n # Public repositories:\n # - Publish results to OpenSSF REST API for easy access by consumers\n # - Allows the repository to include the Scorecard badge.\n # - See https://github.com/ossf/scorecard-action#publishing-results.\n # For private repositories:\n # - `publish_results` will always be set to `false`, regardless\n # of the value entered here.\n publish_results: true\n\n # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF\n # format to the repository Actions tab.\n - name: \"Upload artifact\"\n uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2\n with:\n name: SARIF file\n path: results.sarif\n retention-days: 5\n\n # Upload the results to GitHub's code scanning dashboard.\n - name: \"Upload to code-scanning\"\n uses: github/codeql-action/upload-sarif@28deaeda66b76a05916b6923827895f2b14ab387 # v3.28.16\n with:\n sarif_file: results.sarif\n" }, { "path": ".github/workflows/stale.yml", "content": "---\nname: Mark stale issues and pull requests\non:\n workflow_dispatch:\n schedule:\n - cron: 40 9 * * *\njobs:\n stale:\n uses: secretflow/.github/.github/workflows/stale.yml@main\n" }, { "path": ".github/workflows/trigger-ci-cov.yml", "content": "name: Trigger CircleCI Coverage Test\n\non:\n pull_request_target:\n types: [labeled] # Only run when a label is added\n\njobs:\n trigger-circleci:\n runs-on: ubuntu-latest\n # This job only runs if the label that was just added is exactly \"run-ci-cov\".\n if: github.event.label.name == 'run-ci-cov'\n permissions:\n pull-requests: write\n steps:\n - name: Trigger CircleCI Pipeline\n run: |\n echo \"Label 'run-ci-cov' was added to PR #${{ github.event.pull_request.number }}. Triggering CircleCI pipeline...\"\n\n # We use curl to directly call the CircleCI API v2.\n # This is more flexible than the official trigger action.\n curl -X POST \\\n --url \"https://circleci.com/api/v2/project/github/secretflow/scql/pipeline/run\" \\\n --header \"Content-Type: application/json\" \\\n --header \"Circle-Token: ${{ secrets.CCI_TOKEN }}\" \\\n --data '{\"definition_id\":\"425b19a9-9f21-4d53-a4a6-c2e57939ddd0\",\"config\":{\"branch\":\"pull/${{ github.event.pull_request.number }}/head\"},\"checkout\":{\"branch\":\"pull/${{ github.event.pull_request.number }}/head\"}}'\n - name: Remove Label from PR\n run: |\n echo \"Removing 'run-ci-cov' label from PR #${{ github.event.pull_request.number }}...\"\n\n curl -s -X DELETE \\\n -H \"Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}\" \\\n -H \"Accept: application/vnd.github.v3+json\" \\\n \"https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/labels/run-ci-cov\"\n" }, { "path": ".github/workflows/trigger-ci-full-ut.yml", "content": "name: Trigger CircleCI Full Unit Test for Forked & Labeled PR\n\non:\n pull_request_target:\n types: [labeled]\n\njobs:\n trigger-circleci:\n runs-on: ubuntu-latest\n # This job only runs if the label that was just added is exactly \"run-ci-full\".\n if: github.event.label.name == 'run-ci-full'\n permissions:\n pull-requests: write\n steps:\n - name: Trigger CircleCI Pipeline\n run: |\n echo \"Label 'run-ci-full' was added to PR #${{ github.event.pull_request.number }}. Triggering CircleCI pipeline...\"\n\n curl -X POST \\\n --url \"https://circleci.com/api/v2/project/github/secretflow/scql/pipeline/run\" \\\n --header \"Content-Type: application/json\" \\\n --header \"Circle-Token: ${{ secrets.CCI_TOKEN }}\" \\\n --data '{\"definition_id\":\"ee28d7c2-ef66-4a57-bc90-6895e2b55f14\",\"config\":{\"branch\":\"pull/${{ github.event.pull_request.number }}/head\"},\"checkout\":{\"branch\":\"pull/${{ github.event.pull_request.number }}/head\"}}'\n - name: Remove Label from PR\n run: |\n echo \"Removing 'run-ci-full' label from PR #${{ github.event.pull_request.number }}...\"\n\n curl -s -X DELETE \\\n -H \"Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}\" \\\n -H \"Accept: application/vnd.github.v3+json\" \\\n \"https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/labels/run-ci-full\"\n" }, { "path": ".github/workflows/trigger-ci-lite-ut.yml", "content": "name: Trigger CircleCI Lite Unit Test for Forked & Labeled PR\n\non:\n pull_request_target:\n types: [labeled]\n\njobs:\n trigger-circleci:\n runs-on: ubuntu-latest\n # This job only runs under two conditions:\n # 1. The PR is from a fork.\n # 2. The label that was just added is exactly \"run-ci\".\n if: |\n github.event.pull_request.head.repo.full_name != github.repository &&\n (github.event.action == 'labeled' && github.event.label.name == 'run-ci')\n permissions:\n pull-requests: write\n steps:\n - name: Trigger CircleCI Pipeline\n run: |\n echo \"Forked PR #${{ github.event.pull_request.number }} has 'run-ci' label. Triggering CircleCI pipeline...\"\n\n curl -X POST \\\n --url \"https://circleci.com/api/v2/project/github/secretflow/scql/pipeline/run\" \\\n --header \"Content-Type: application/json\" \\\n --header \"Circle-Token: ${{ secrets.CCI_TOKEN }}\" \\\n --data '{\"definition_id\":\"c6f435ca-697f-488c-83da-156862bb7392\",\"config\":{\"branch\":\"pull/${{ github.event.pull_request.number }}/head\"},\"checkout\":{\"branch\":\"pull/${{ github.event.pull_request.number }}/head\"}}'\n - name: Remove Label from PR\n run: |\n echo \"Removing 'run-ci' label from PR #${{ github.event.pull_request.number }}...\"\n\n curl -s -X DELETE \\\n -H \"Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}\" \\\n -H \"Accept: application/vnd.github.v3+json\" \\\n \"https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/labels/run-ci\"\n" }, { "path": ".github/workflows/trigger-diff-coverage.yml", "content": "name: Manually Trigger Incremental Coverage Test\n\n# Use workflow_dispatch to allow manual triggering\non:\n workflow_dispatch:\n inputs:\n base_commit:\n description: 'The base Commit SHA for comparison (the older one)'\n required: true\n head_commit:\n description: 'The head Commit SHA to generate coverage for (the newer one)'\n required: true\n\njobs:\n trigger-circleci-diff-coverage:\n runs-on: ubuntu-latest\n steps:\n - name: \"Trigger CircleCI Incremental Coverage Pipeline via API\"\n run: |\n echo \"Triggering CircleCI pipeline for base=${{ github.event.inputs.base_commit }} and head=${{ github.event.inputs.head_commit }}...\"\n\n curl --request POST \\\n --url \"https://circleci.com/api/v2/project/github/secretflow/scql/pipeline/run\" \\\n --header \"Content-Type: application/json\" \\\n --header \"Circle-Token: ${{ secrets.CCI_TOKEN }}\" \\\n --data '{\"definition_id\":\"f0835e5e-5836-4869-bcb6-f51ff967281d\",\"checkout\":{\"branch\":\"${{ github.event.inputs.head_commit }}\"},\"parameters\":{\"base_commit\":\"${{ github.event.inputs.base_commit }}\"},\"config\":{\"branch\":\"${{ github.event.inputs.head_commit }}\"}}'\n" }, { "path": ".github/workflows/whitespace-check.yml", "content": "name: Whitespace Check\n\non:\n pull_request:\n types: [opened, synchronize]\n\nconcurrency:\n group: ${{ github.workflow }}-${{ github.ref }}\n cancel-in-progress: true\n\njobs:\n check-whitespace:\n runs-on: ubuntu-latest\n steps:\n - name: Checkout code\n uses: actions/checkout@v4\n with:\n fetch-depth: 0\n\n - name: Check for whitespace issues\n run: |\n ./scripts/check-whitespace.sh \\\n \"${{ github.event.pull_request.base.sha }}\" \\\n \"$GITHUB_STEP_SUMMARY\" \\\n \"https://github.com/${{ github.repository }}\"\n" }, { "path": ".github/workflows/yaml-linter.yml", "content": "---\nname: Yaml Lint\non:\n push:\n branches:\n - main\n pull_request:\n branches:\n - main\npermissions:\n contents: read\njobs:\n yaml-linter:\n uses: secretflow/.github/.github/workflows/yaml-linter.yml@main" }, { "path": ".gitignore", "content": "# docs\ndocs/_build\n\n# bazel\n/bazel-*\n\n# go\nbin/*\ntool-bin/*\n/vendor\n\n# cloudide\n.cloudide/\n\n/compile_commands.json\n\n# clangd cache\n/external\n/.cache/\n.vscode/launch.json\n\n# Translations\n*.mo\n*.pot\n\n*.pyc\n\n.venv\n\n# 排除 ide 文件\n.idea/\n# 排除mac本地文件\n*.DS_Store\n\n# ignore bazelrc for remote cache\n.remote.bazelrc\n\n# python binding build\npython/build/\npython/package/\n" }, { "path": ".golangci.yml", "content": "# This file contains all available configuration options\n# with their default values.\nversion: \"2\"\n\n# options for analysis running\nrun:\n # default concurrency is a available CPU number\n concurrency: 4\n\n # timeout for analysis, e.g. 30s, 5m, default is 1m\n timeout: 10m\n\n # exit code when at least one issue was found, default is 1\n issues-exit-code: 1\n\n # include test files or not, default is true\n tests: true\n\n # list of build tags, all linters use it. Default is empty list.\n build-tags: []\n\n# output configuration options\noutput:\n # sort order\n sort-order:\n - linter\n - severity\n - file\n\n # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is \"colored-line-number\"\n formats:\n text:\n path: stdout\n print-linter-name: true\n print-issued-lines: true\n colors: true\n\nissues:\n max-same-issues: 0\n max-issues-per-linter: 0\n\nformatters:\n enable:\n - gofmt\n - goimports\n settings:\n gofmt:\n # simplify code: gofmt with `-s` option, true by default\n simplify: true\n goimports:\n # put imports beginning with prefix after 3rd-party packages;\n # it's a comma-separated list of prefixes\n local-prefixes:\n - github.com/secretflow/scql\n\nlinters:\n exclusions:\n paths:\n - _test\\.go\n - pkg/planner\n - pkg/parser\n - pkg/proto-gen\n - pkg/expression\n - pkg/types\n - pkg/util\n rules:\n - text: 'shadow: declaration of \"(err|ctx)\" shadows declaration at'\n linters: [govet]\n settings:\n staticcheck:\n # All supported checks can be enabled with \"all\".\n # To disable checks, prefix them with a minus sign.\n # Example: [ \"all\", \"-SA1000\", \"-SA1001\"]\n checks: [\"all\", \"-ST1000\", \"-ST1003\", \"-ST1016\", \"-ST1020\", \"-ST1021\", \"-ST1022\", \"-QF1003\", \"-QF1006\", \"-QF1008\"]\n gosec:\n excludes:\n - G115 # Potential integer overflow when converting between integer types\n - G401 # Detect the usage of MD5 or SHA1\n - G402 # Look for bad TLS connection settings\n - G501 # Import blocklist: crypto/md5\n revive:\n rules:\n - name: unused-parameter # disable unused-parameter rule\n disabled: true\n errcheck:\n # report about not checking of errors in type assetions: `a := b.(MyStruct)`;\n # default is false: such cases aren't reported by default.\n check-type-assertions: false\n\n # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`;\n # default is false: such cases aren't reported by default.\n check-blank: false\n\n exclude-functions:\n - fmt.Fprintf(os.Stdout)\n - fmt.Fprintln(os.Stdout)\n - (*database/sql.Rows).Close # prefix is required\n - (*os.File).Close\n - (io.ReadCloser).Close\n - (io.Close).Close\n - (*google.golang.org/grpc.ClientConn).Close\n govet:\n # report about shadowed variables\n enable:\n - shadow\n gocyclo:\n # minimal code complexity to report, 30 by default (but we recommend 10-20)\n min-complexity: 10\n dupl:\n # tokens count to trigger issue, 150 by default\n threshold: 150\n goconst:\n # minimal length of string constant, 3 by default\n min-len: 3\n # minimal occurrences count to trigger, 3 by default\n min-occurrences: 3\n depguard:\n rules:\n main:\n deny:\n - pkg: \"github.com/davecgh/go-spew/spew\"\n misspell:\n # Correct spellings using locale preferences for US or UK.\n # Default is to use a neutral variety of English.\n # Setting locale to US will correct the British spelling of 'colour' to 'color'.\n locale: US\n ignore-rules:\n - someword\n lll:\n # max line length, lines longer will be reported. Default is 120.\n # '\\t' is counted as 1 character by default, and can be changed with the tab-width option\n line-length: 120\n # tab width in spaces. Default to 1.\n tab-width: 1\n unused:\n parameters-are-used: true\n exported-fields-are-used: true\n unparam:\n # Inspect exported functions, default is false. Set to true if no external program/library imports your code.\n # XXX: if you enable this setting, unparam will report a lot of false-positives in text editors:\n # if it's called for subdir of a project it can't find external interfaces. All text editor integrations\n # with golangci-lint call it on a directory with the changed file.\n check-exported: false\n nakedret:\n # make an issue if func has more lines of code than this setting and it has naked returns; default is 30\n max-func-lines: 30\n prealloc:\n # XXX: we don't recommend using this linter before doing performance profiling.\n # For most programs usage of prealloc will be a premature optimization.\n\n # Report preallocation suggestions only on simple loops that have no returns/breaks/continues/gotos in them.\n # True by default.\n simple: true\n range-loops: true # Report preallocation suggestions on range loops, true by default\n for-loops: false # Report preallocation suggestions on for loops, false by default\n gocritic:\n # Which checks should be disabled; can't be combined with 'enabled-checks'; default is empty\n disabled-checks:\n - regexpMust\n\n # Enable multiple checks by tags, run `GL_DEBUG=gocritic golangci-lint` run to see all tags and checks.\n # Empty list by default. See https://github.com/go-critic/go-critic#usage -> section \"Tags\".\n enabled-tags:\n - performance\n\n settings: # settings passed to gocritic\n captLocal: # must be valid enabled check name\n paramsOnly: true\n enable:\n - unused\n - govet\n - revive\n - errcheck\n - goconst\n - dupl\n # - gosec\n - staticcheck\n - bodyclose\n default: none\n" }, { "path": ".licenserc.yaml", "content": "header:\n license:\n spdx-id: Apache-2.0\n copyright-owner: Ant Group Co., Ltd.\n copyright-year: auto\n software-name: secretflow\n\n pattern: |\n Licensed under the Apache License, Version 2.0 \\(the \"License\"\\);\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http[s]?://www\\.apache\\.org/licenses/LICENSE-2\\.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n paths:\n - \"**\"\n\n paths-ignore:\n - \".aci\"\n - \".ci\"\n - \".circleci\"\n - \".github\"\n - \".vscode\"\n - \"benchmark/docker-compose/broker\"\n - \"benchmark/docker-compose/engine\"\n - \"benchmark/docker-compose/mysql\"\n - \"benchmark/testdata\"\n - \"contrib/agent/kusciastub/mock_kusciaapi/mock_kusciaapi.go\"\n - \"docs/_static\"\n - \"docs/imgs\"\n - \"examples\"\n - \"pkg/executor/engine_stub_mock.go\"\n - \"pkg/expression\"\n - \"pkg/infoschema\"\n - \"pkg/parser\"\n - \"pkg/planner/core\"\n - \"pkg/planner/property\"\n - \"pkg/planner/util/path.go\"\n - \"pkg/privilege\"\n - \"pkg/sessionctx\"\n - \"pkg/table\"\n - \"pkg/types\"\n - \"pkg/util/chunk\"\n - \"pkg/util/codec\"\n - \"pkg/util/execdetails\"\n - \"pkg/util/hack\"\n - \"pkg/util/math\"\n - \"pkg/util/mathutil\"\n - \"pkg/util/misc.go\"\n - \"pkg/util/mvmap\"\n - \"pkg/util/plancodec\"\n - \"pkg/util/ranger\"\n - \"pkg/util/stringutil\"\n - \"pkg/util/testleak\"\n - \"pkg/util/testutil\"\n - \"pkg/util/texttree\"\n - \"scripts\"\n - \"**/*.template\"\n - \"**/*.patch\"\n - \"**/*.pb.go\"\n - \"**/*.md\"\n - \"**/*.json\"\n - \"**/*.yaml\"\n - \"**/*.yml\"\n - \"**/*.txt\"\n - \"**/*.Dockerfile\"\n - \"**/Dockerfile\"\n - \"**/*.po\"\n - \"**/*.svg\"\n - \"**/*.bat\"\n - \"**/*.rst\"\n - \"**/*.patch\"\n - \"**/*.csv\"\n - \"**/*.tmpl\"\n - \"LICENSE\"\n - \"NOTICE\"\n - \"MODULE.bazel.lock\"\n - \".bazeliskrc\"\n - \".bazelrc\"\n - \".clang-format\"\n - \".clang-tidy\"\n - \"**/.gitignore\"\n - \"**/.env\"\n - \"go.mod\"\n - \"go.sum\"\n - \"**/Makefile\"\n\n comment: never\n\n license-location-threshold: 80\n\n language:\n Starlark:\n extensions:\n - \".bazel\"\n - \".bazelrc\"\n - \"BUILD\"\n - \".bzl\"\n - \"WORKSPACE\"\n comment_style_id: PythonStyle\n Cpp:\n extensions:\n - \".cc\"\n - \".h\"\n - \".cu\"\n comment_style_id: DoubleSlash\n Python:\n extensions:\n - \".py\"\n comment_style_id: PythonStyle\n" }, { "path": ".markdownlint.yaml", "content": "# Copyright 2025 Ant Group Co., Ltd.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# Enable all markdownlint rules by default\ndefault: true\nMD013: false # line-length Line length\nMD024: false # no-duplicate-heading Multiple headings with the same content\n\n# List formatting:\n#\n# 1. Lists must have a blank line before AND after the list.\n# 2. Lists start aligned to the left (do not indent the top level list items).\n# NOTE: markdownlint currently checks indentation for unordered lists only.\n# Please manually verify that your ordered lists are not indented.\n# See https://github.com/DavidAnson/markdownlint/issues/138.\n# 3. You may use one or zero blank lines between list items.\n# 4. Nested list items should be indented to align with the first character of\n# the first line. For bullet lists, that means 2 spaces. For numbered\n# lists, that's 3 spaces (but 4 spaces is okay if that's easier).\n# 5. In multiline list items, subsequent lines are indented by 2 spaces.\n# This is not checked automatically, so we're documenting this convention\n# to make sure the codebase stays consistent.\n#\n# Examples:\n#\n# * This is a list item that has multiple\n# lines and each line aligns with the text from the first line.\n# * This is a nested list, also aligned with the first line.\n#\n# For ordered lists, that means three spaces for wrapped lines:\n#\n# 1. This is an ordered list item.\n# 1. The nested list aligns with the first line.\nul-indent:\n indent: 2\n\n# Allow inline HTML\nno-inline-html: false\n\n# Allow dupe heading names only if they're not siblings\nno-duplicate-heading:\n siblings_only: true\n\n# Allow images w/o alt-text\nno-alt-text: false" }, { "path": ".pre-commit-config.yaml", "content": "repos:\n - repo: https://github.com/gitleaks/gitleaks\n rev: v8.24.0\n hooks:\n - id: gitleaks\n - repo: https://github.com/golangci/golangci-lint\n rev: v2.0.0\n hooks:\n - id: golangci-lint\n - repo: https://github.com/jumanjihouse/pre-commit-hooks\n rev: 3.0.0\n hooks:\n - id: shellcheck\n - repo: https://github.com/pocc/pre-commit-hooks\n rev: v1.3.5\n hooks:\n - id: cpplint\n - repo: https://github.com/pre-commit/pre-commit-hooks\n rev: v4.4.0\n hooks:\n - id: end-of-file-fixer\n - id: trailing-whitespace\n - repo: https://github.com/pylint-dev/pylint\n rev: v2.17.2\n hooks:\n - id: pylint\n" }, { "path": ".vscode/cspell.json", "content": "// cSpell Settings\n{\n // Version of the setting file. Always 0.2\n \"version\": \"0.2\",\n // language - current active spelling language\n \"language\": \"en\",\n // words - list of words to be always considered correct\n \"words\": [\n \"brokerctl\",\n \"gorm\",\n \"Kuscia\",\n \"kusciaapi\",\n \"protojson\",\n \"quickstart\",\n \"scql\",\n \"scqlengine\",\n \"secretflow\",\n \"spu\",\n \"varchar\"\n ]\n}\n" }, { "path": ".vscode/extensions.json", "content": "{\n \"recommendations\": [\n \"BazelBuild.vscode-bazel\",\n \"eamodio.gitlens\",\n \"golang.go\",\n \"ms-python.python\",\n \"ms-vscode.cpptools\",\n \"jgclark.vscode-todo-highlight\",\n \"ms-vscode-remote.remote-ssh\",\n \"drblury.protobuf-vsc\",\n \"llvm-vs-code-extensions.vscode-clangd\",\n \"lextudio.restructuredtext\",\n \"bufbuild.vscode-buf\"\n ]\n}\n" }, { "path": ".vscode/settings.json", "content": "{\n \"editor.formatOnSave\": true,\n \"gopls\": {\n \"formatting.local\": \"github.com/secretflow/scql\"\n },\n \"[python]\": {\n \"editor.defaultFormatter\": \"ms-python.black-formatter\"\n },\n \"files.watcherExclude\": {\n \"**/.git/**\": true,\n \"**/.cache/**\": true,\n \"**/bazel-*/**\": true,\n \"**/external/**\": true\n },\n \"files.insertFinalNewline\": true,\n \"files.trimTrailingWhitespace\": true,\n \"[proto]\": {\n \"editor.defaultFormatter\": \"bufbuild.vscode-buf\",\n \"editor.formatOnSave\": true\n }\n}\n" }, { "path": "BUILD.bazel", "content": "# Copyright 2023 Ant Group Co., Ltd.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n" }, { "path": "CHANGELOG.md", "content": "# Changelog\n\nAll notable changes to this project will be documented in this file.\n\nThe format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),\nand this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).\n\n## Types of changes\n\n`Added` for new features.\n`Changed` for changes in existing functionality.\n`Deprecated` for soon-to-be removed features.\n`Removed` for now removed features.\n`Fixed` for any bug fixes.\n`Security` in case of vulnerabilities.\n\n## Staging\n\n## [2.0.0] - 2026-03-16\n\n### Added\n\n- Introduced SCQL 2.0 OpenCore architecture: new `CompilerService` gRPC API (`api/v1alpha1/compiler.proto`) with `CompileSQL` endpoint, and a new Go compiler package (`pkg/interpreter/compiler/`) replacing the old interpreter pipeline.\n- Added Perfetto-based tracing support in engine, controlled by `--enable_trace` and `--trace_log_path` flags.\n- Added `GroupSecretSum` and `GroupSecretAvg` operators using SPU secret-sharing, replacing the old HE-based group aggregation. Benchmarks show the new algorithm is 5-10x faster.\n- Added planner optimization rules: correlated subquery decorrelation, group-by threshold enforcement, `JOIN` reorder by party code, and consecutive selection merging.\n- Added `Rr22Mode` (low/fast) for PSI algorithm negotiation.\n- Added `opencore-demo` example and quickstart documentation for getting started with the new architecture.\n\n### Changed\n\n- **breaking**: Redesigned system architecture from Broker-based (P2P/Centralized) to native Compiler + Engine integration.\n- **breaking**: `Executor.RunExecutionPlan()` now returns `*scql.QueryResponse` instead of `*scql.SCDBQueryResultResponse`.\n- Refactored engine operators to use typed `ExecContext` accessor methods (`GetInputTensor()`, `SetOutputTensor()`, etc.) instead of manual tensor table lookups.\n- Refactored session negotiation: new `Session::Negotiate()` method handles streaming options, PSI options, and curve types via protobuf-serialized `NegotiationOptions`.\n- Consolidated `StreamingOptions` into `SessionOptions`.\n- Extracted `RunPlanCore` from `EngineServiceImpl` into a standalone function for reuse in both RPC and task modes.\n- Updated documentation to reflect the OpenCore architecture.\n\n### Removed\n\n- Removed SCDB centralized query server (`pkg/scdb/`, `cmd/scdbserver/`, `cmd/scdbclient/`, `api/scdb_api.proto`).\n- Removed Broker P2P coordination layer (`pkg/broker/`, `cmd/broker/`, `cmd/brokerctl/`, `api/broker.proto`).\n- Removed CCL (Column Control Language) authorization mechanism (`api/ccl.proto`, `pkg/interpreter/ccl/`).\n- Removed old interpreter/translator pipeline (`pkg/interpreter/translator/`, `pkg/interpreter/interpreter.go`, `pkg/interpreter/svc/`).\n- Removed `cmd/regtest/`, `cmd/agent/`, `cmd/scqltool/`, `pkg/privilege/`, `pkg/executor/job_watcher.go`.\n- Removed `GroupHESum` operator (replaced by `GroupSecretSum`/`GroupSecretAvg`).\n- Removed all legacy deployment guides, API references, CCL docs, and CI infrastructure for removed components.\n\n### Fixed\n\n- Fixed communication stats not accounting for initialization-phase network traffic.\n- Fixed potential async task lifecycle issues in `RunPlanSync` by switching from `std::async` to direct invocation.\n- Fixed null handling in window function, placing null values at the end.\n- Fixed missing `DESC` keyword when rebuilding ORDER BY statement.\n- Fixed interpreter creation time losing timezone information.\n\n## [0.9.4] - 2025-07-29\n\n### Added\n\n- Added Archive API to support project archiving.\n- Enhanced time-type data processing capability: support `STR_TO_DATE` function and implicit conversion from string to time types.\n- Supported richer expression of statements: `PERCENTILE_DISC`, `BETWEEN AND`, `REPLACE`, etc.\n\n### Changed\n\n- Optimized data source reading, improving streaming processing capabilities.\n- Optimized `JOIN` process to eliminate the need for additional `PLAINTEXT_AFTER_JOIN` CCL for non-result-receiving parties on join keys.\n\n### Fixed\n\n- Resolved the issue with compare subquery exceptions in aggregation scenarios.\n- Fixed column disorder issues in the project tables.\n- Resolved problems in LogicalOptimizer to prevent the removal of LogicalProjection nodes that could lead to performance and Tensor property inference issues.\n\n## [0.9.3] - 2025-03-07\n\n### Added\n\n- Support for datasource `Doris 2.1.7`.\n- Support `PERCENT_RANK` window function.\n- Support various string-related single-party operators, including `UPPER`, `LOWER`, `SUBSTRING`, `TRIM`, `CONCAT` and others.\n- Support `Scalar Subquery`, the subquery in the right is scalar value, e.g. SELECT * FROM ta JOIN tb ON ta.ID = tb.ID WHERE ta.salary > (SELECT AVG(ta.salary) FROM ta).\n- Support `Compare Subquery`, allows comparison with ANY or ALL of the subquery results, e.g. SELECT * FROM ta JOIN tb ON ta.ID = tb.ID WHERE ta.salary > ANY(SELECT ta.salary FROM ta), However, comparisons using = or != are not supported in the HAVING clause. For instance, HAVING SUM(ta.salary) = ANY(SELECT salary FROM ta) is not supported.\n\n### Changed\n\n- Improved `JOIN` and `IN` performance in streaming mode.\n- Implemented a more reliable `secret join algorithm`(only works in SEMI2K protocol) inspired by [Scape](https://ieeexplore.ieee.org/document/9835540/).\n- Optimized the column pruning rule for Join, Selection, and Window nodes in the Logical Optimizer to more effectively remove redundant columns.\n\n### Fixed\n\n- Restricted access to SCQLEngine metrics using additional paths like \"engine_ip:engine_port/metrics/additional/path\".\n- Prevented creation of tables with the same ref_table name but different db_type\n- Fixed job creation error when selecting 'OPRF-PSI' but 'server hint' was missing.\n\n## [0.9.2] - 2024-12-23\n\n### Added\n\n- Enhancement: Support `JOIN` after `UNION` operation.\n- Add SCQL Agent to facilitate running SCQL query tasks in Kuscia, making it easier to integrate into SecretPad.\n- Support writing results into multi-parties via `SELECT INTO OUTFILE` syntax.\n- Support datasource `ODPS` via integrating with [dataproxy](https://github.com/secretflow/dataproxy).\n- Support `order by`.\n- Support a lot of single-party operators, such as `ABS`, `ASIN`, `EXP`, `FLOOR`, `SQRT` etc.\n\n### Changed\n\n- Improve the `JOIN` and `IN` performance via integrating [RR22 PSI](https://github.com/secretflow/psi/blob/v0.5.0b0/psi/proto/psi_v2.proto#L62).\n- Improve the aggregation with group by performance if `reveal_group_count` enabled.\n\n### Fixed\n\n- Fixed an occasional crash issue when canceling query job.\n- Fixed `select now()` is not supported issue.\n\n## [0.9.1] - 2024-10-16\n\n### Added\n\n- Support window function `ROW_NUMBER()` with PARTITION BY clause and ORDER BY clause.\n- Add new CCL constraint `REVAL_RANK`.\n- Add ExplainQuery API with path `/intra/query/explain`.\n- Support `INSERT INTO SELECT` syntax to allow writing query result back to db (mysql/sqlite/postgres).\n- Support `trim` function.\n\n### Changed\n\n- Improved the job watcher to work better in broker clustered mode.\n\n## [0.9.0] - 2024-08-01\n\n### Added\n\n- Support write outfile to OSS/MINIO via `select into` query.\n- Support `sin`, `cos`, `acos` function.\n- Support `geodist` function.\n- Broker support using postgres as metadata storage.\n\n### Changed\n\n- Reduce the memory peak of large-scale intersection tasks through streaming execution.\n- Link tcmalloc to solve the problem of memory increase.\n\n### Fixed\n\n- Fix crashes when dumpfile exceeds 2GB string column.\n- Reduce the probability of graph checksum inconsistency issues.\n\n## [0.8.1] - 2024-07-02\n\n### Added\n\n- Support session-based log isolation functioality in the SCQL Engine.\n- Support consul-based broker registration/discovery services, providing ACL/TLS authentication.\n\n### Changed\n\n### Fixed\n\n## [0.8.0] - 2024-06-12\n\n### Added\n\n- Enhanced `FetchResult` RPC and `brokerctl get result` command to report job progress when result is not ready.\n- Support project/query level configs\n- Support NULL for private data: including Arithmetic, Logic, Aggregation, etc., {IS [NOT] NULL, IFNULL, COALESCE} are also supported.\n- Support port isolation for engine link service and control panel service (RunExecutionPlan).\n- Add new CCL constraint `PLAINTEXT_AS_JOIN_PAYLOAD`.\n\n### Changed\n\n- **breaking**: The response value type of Broker API `DoQuery` and `FetchResult` have incompatible changes.\n\n### Fixed\n\n## [0.7.0] - 2024-05-14\n\n### Added\n\n- Added CheckAndUpdate API for self-recovery when status is inconsistent in P2P mode.\n\n### Fixed\n\n- Fixed the problem that Broker was unable to detect SCQLEngine crashes or being killed by OOM.\n\n## [0.6.0] - 2024-04-15\n\n### Added\n\n- Support for RSA key pairs in SCQLBroker.\n- Support running on [kuscia](https://github.com/secretflow/kuscia) and scheduling SCQLEngine dynamic via kuscia job.\n- Added `dry_run` parameter in DoQuery request, it could be used to check query syntax and CCL without actually executing the query.\n- Improve Broker high availability, support deploying in multi-node cluster deployment.\n- Support reading csv from OSS/MINIO.\n\n### Changed\n\n- **breaking**: Reshape column data type, data type `LONG` is deprecated.\n- **breaking**: Modify table schema in broker storage for P2P mode.\n\n## [0.5.0] - 2024-01-10\n\n### Added\n\n- Added support for HTTP data source router.\n\n### Changed\n\n- **breaking**: Add table **members** in broker storage for P2P mode.\n- Speed up GROUP BY with Radix Sort.\n- Adjusted configuration items for SCQLEngine and SCQLBroker.\n\n### Fixed\n\n- Fixed check for grant ccl in P2P mode.\n\n## [0.4.0] - 2023-11-15\n\n### Added\n\n- Added support for P2P mode, no longer need to rely on a trusted third party.\n- Added support for {datetime, timestamp} data types, as well as related operations.\n- Support using ArrowSQL as a data source for Engine\n- Added support for {Limit Cast Mod} operators.\n\n### Changed\n\n- Polished document outline.\n\n## [0.3.0] - 2023-09-10\n\n### Added\n\n- Optimize SCQLEngine memory usage, release unused tensors immediately.\n- Added warning information to the query result.\n- Added support for {LEFT JOIN, RIGHT JOIN, CASE WHEN, IF} operators\n\n### Changed\n\n- Speed up GROUP BY with HEU in some scenarios.\n- Optimized to support billion-level PSI scenarios.\n- Drop GRM from SCQL awareness. We extend the syntax of create user statement and modify the syntax of create table statement.\n- Used json string format to configure spu runtime in scdb yaml conf.\n- Speed up JOIN, IN with Unbalanced PSI in scenarios with unbalanced data.\n\n## [0.2.0] - 2023-06-30\n\n### Added\n\n- Added support for union operator.\n- Added support for reading CSV files as a data source for Engine.\n- Added support for using PostgreSQL as a database for Engine.\n- Added support for change password with ALTER USER statement.\n- Added support for removing table-level and database-level permissions with the REVOKE statement.\n- Added support for structured audit log.\n- Added support for the float64 data type in the Engine.\n- Added the Chinese documentation.\n\n### Changed\n\n- Change some description in document.\n- Enrich test cases.\n- Enhanced support for security protocols of Cheetah and ABY3.\n- Optimized GROUP BY logic.\n- Optimized execution plan nodes.\n- Optimized the execution logic of the runSQL.\n- Optimized the three party ccl in join node.\n\n### Fixed\n\n- Fixed create database failed [#19](https://github.com/secretflow/scql/issues/19).\n- Fixed not support group by string[#48](https://github.com/secretflow/scql/pull/48).\n\n## [0.1.0] - 2023-03-28\n\n### Added\n\n- SCQL init release\n" }, { "path": "CONTRIBUTING.md", "content": "# Contributing\n\n## Contributor License Agreement\n\nContributions to this project must be accompanied by a Contributor License\nAgreement. You (or your employer) retain the copyright to your contribution;\nthis simply gives us permission to use and redistribute your contributions as\npart of the project.\n\n## Repo layout\n\n- Please see [repo layout](REPO_LAYOUT.md).\n\n## Style\n\n### Go coding style\n\nGo code follows [Uber Go Style Guide](https://github.com/uber-go/guide/blob/master/style.md)\n\n### C++ coding style\n\nIn general, please use clang-format to format code, and follow clang-tidy tips.\n\nMost of the code style is derived from the [Google C++ style guidelines](https://google.github.io/styleguide/cppguide.html), except:\n\n- Exceptions are allowed and encouraged where appropriate.\n- Header guards should use `#pragma once`.\n\n### Other tips\n\n- Git commit message should be meaningful, we suggest imperative [keywords](https://github.com/joelparkerhenderson/git_commit_message#summary-keywords).\n- Developer must write unit-test (line coverage must be greater than 80%), tests should be deterministic.\n\n## Build\n\n### Prerequisite\n\n#### Docker\n\n\n```sh\n## start dev container\ndocker run -d -it --name scql-dev-$(whoami) \\\n --mount type=bind,source=\"$(pwd)\",target=/home/admin/dev/ \\\n -w /home/admin/dev \\\n --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \\\n --cap-add=NET_ADMIN \\\n --privileged=true \\\n secretflow/scql-ci:latest /bin/bash\n\n# attach to dev container\ndocker exec -it scql-dev-$(whoami) bash\n```\n\n### Build & UnitTest\n\n\n\n\n```sh\n\n# build SCQL engine as release\nbazelisk build //engine/exe:scqlengine -c opt\n\n# run unittests for SCQL engine\nbazelisk test //engine/...\n\n# build with address sanitizer\nbazelisk build --config=asan //engine/exe:scqlengine\n\n# build go code\nmake\n\n# run go unit tests\ngo test ./pkg/...\n```\n\n### Build docs\n\n```sh\n# prerequisite\npip3 install -U -r docs/requirements.txt\n\n# Build HTML docs, and the result is placed in directory 'docs/_build/html'\n# Build documentation in English and Chinese\nmake doc\n```\n" }, { "path": "LEGAL.md", "content": "# Legal Disclaimer\n\nWithin this source code, the comments in Chinese shall be the original, governing version. Any comment in other languages are for reference only. In the event of any conflict between the Chinese language version comments and other language version comments, the Chinese language version shall prevail.\n\n法律免责声明\n\n关于代码注释部分，中文注释为官方版本，其它语言注释仅做参考。中文注释可能与其它语言注释存在不一致，当中文注释与其它语言注释存在不一致时，请以中文注释为准。\n" }, { "path": "LICENSE", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "MODULE.bazel", "content": "# Copyright 2024 Ant Group Co., Ltd.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nmodule(\n name = \"scql\",\n version = \"0.9.5dev\",\n compatibility_level = 1,\n)\n\nbazel_dep(name = \"yacl\", version = \"0.4.5b12-nightly-20251024\")\nbazel_dep(name = \"psi\", version = \"0.6.0.dev250922\")\nbazel_dep(name = \"spulib\", version = \"0.9.5rc1\")\ngit_override(\n module_name = \"spulib\",\n commit = \"2387c999a4bd2b7e85072f4e25a173a60b5958d4\",\n remote = \"https://github.com/secretflow/spu.git\",\n strip_prefix = \"src\",\n)\n\nbazel_dep(name = \"dataproxy_sdk_cc\", version = \"0.5.0.dev20250603\")\nbazel_dep(name = \"kuscia\", version = \"0.13.0b1\")\nbazel_dep(name = \"apple_support\", version = \"1.24.5\")\nbazel_dep(name = \"rules_cuda\", version = \"0.2.3\")\nbazel_dep(name = \"rules_cc\", version = \"0.0.12\")\nbazel_dep(name = \"rules_proto\", version = \"6.0.2\")\n\n# non module dependencies\nnon_module_dependencies = use_extension(\"//bazel:defs.bzl\", \"non_module_dependencies\")\nuse_repo(\n non_module_dependencies,\n \"com_github_duckdb\",\n \"com_github_gperftools_gperftools\",\n \"org_pocoproject_poco\",\n)\n\nbazel_dep(name = \"arrow\", version = \"17.0.0.bcr.1\", repo_name = \"org_apache_arrow\")\nbazel_dep(name = \"ncurses\", version = \"6.4.20221231.bcr.3\")\nbazel_dep(name = \"rules_pkg\", version = \"1.0.1\")\nbazel_dep(name = \"rules_java\", version = \"8.6.1\")\nbazel_dep(name = \"rules_go\", version = \"0.53.0\")\nbazel_dep(name = \"rules_proto_grpc_cpp\", version = \"5.0.1\")\nbazel_dep(name = \"rules_foreign_cc\", version = \"0.15.0\")\nsingle_version_override(\n module_name = \"rules_foreign_cc\",\n patch_strip = 1,\n patches = [\n \"//bazel/patches:rules_foreign_cc.patch\",\n ],\n version = \"0.15.0\",\n)\n\nbazel_dep(name = \"bazel_features\", version = \"1.20.0\")\nbazel_dep(name = \"platforms\", version = \"0.0.8\")\nbazel_dep(name = \"openssl\", version = \"3.3.2.bcr.1\")\nbazel_dep(name = \"spdlog\", version = \"1.14.1\")\nbazel_dep(name = \"fmt\", version = \"11.0.2\")\nbazel_dep(name = \"brpc\", version = \"1.13.0\")\nbazel_dep(name = \"abseil-cpp\", version = \"20240722.0\")\nbazel_dep(name = \"boost.uuid\", version = \"1.83.0.bcr.1\")\nbazel_dep(name = \"boost.multiprecision\", version = \"1.83.0\")\nbazel_dep(name = \"boost.endian\", version = \"1.83.0\")\nbazel_dep(name = \"boost.geometry\", version = \"1.83.0\")\nbazel_dep(name = \"boost.graph\", version = \"1.83.0\")\nbazel_dep(name = \"boost.spirit\", version = \"1.83.0\")\nbazel_dep(name = \"bazel_skylib\", version = \"1.7.1\")\nbazel_dep(name = \"boost.serialization\", version = \"1.83.0.bcr.1\")\nbazel_dep(name = \"prometheus-cpp\", version = \"1.2.4\")\nbazel_dep(name = \"protobuf\", version = \"27.3\", repo_name = \"com_google_protobuf\")\nsingle_version_override(\n module_name = \"protobuf\",\n version = \"27.3\",\n)\n\nbazel_dep(name = \"perfetto\", version = \"41.0\")\nbazel_dep(name = \"googleapis\", version = \"0.0.0-20240819-fe8ba054a\")\nbazel_dep(name = \"zlib\", version = \"1.3.1.bcr.3\")\nbazel_dep(name = \"rules_python\", version = \"0.34.0\")\nbazel_dep(name = \"gflags\", version = \"2.2.2\")\nbazel_dep(name = \"msgpack-c\", version = \"6.1.0\")\nbazel_dep(name = \"grpc\", version = \"1.66.0.bcr.4\")\nbazel_dep(name = \"pybind11_bazel\", version = \"3.0.0\")\n\nsingle_version_override(\n module_name = \"grpc\",\n version = \"1.66.0.bcr.4\",\n)\n\nsingle_version_override(\n module_name = \"openssl\",\n version = \"3.3.2.bcr.1\",\n)\n\n# use sf.bcr\nbazel_dep(name = \"curl\", version = \"8.4.0.bcr.2\")\nsingle_version_override(\n module_name = \"curl\",\n version = \"8.4.0.bcr.2\",\n)\n\nnew_local_repository = use_repo_rule(\"@bazel_tools//tools/build_defs/repo:local.bzl\", \"new_local_repository\")\n\nnew_local_repository(\n name = \"macos_omp_x64\",\n build_file = \"@yacl//bazel:local_openmp_macos.BUILD\",\n path = \"/usr/local/opt/libomp\",\n)\n\nnew_local_repository(\n name = \"macos_omp_arm64\",\n build_file = \"@yacl//bazel:local_openmp_macos.BUILD\",\n path = \"/opt/homebrew/opt/libomp/\",\n)\n\npython = use_extension(\"@rules_python//python/extensions:python.bzl\", \"python\")\npython.toolchain(\n ignore_root_user_error = True,\n python_version = \"3.11\",\n)\n\n# test\nbazel_dep(name = \"googletest\", version = \"1.15.2\", dev_dependency = True)\nbazel_dep(name = \"google_benchmark\", version = \"1.8.5\", dev_dependency = True)\nbazel_dep(name = \"rules_buf\", version = \"0.4.0\")\n" }, { "path": "Makefile", "content": "export GO111MODULE=on\nGOPATH := ${GOPATH}:${PWD}\nTOOLBIN := ${PWD}/tool-bin\nexport PATH := ${TOOLBIN}:$(PATH)\nexport GOFLAGS=-buildmode=pie -buildvcs=false\nexport CGO_CPPFLAGS=-fstack-protector-strong -D_FORTIFY_SOURCE=2\nUNAME_S := $(shell uname -s)\nifeq ($(UNAME_S),Linux)\n\texport CGO_LDFLAGS=-Wl,-z,relro,-z,now,-z,noexecstack\nendif\n\n.PHONY: clean vet lint test detect-shadowing fast pb prepare fmt gogenerate\n\ndefault: install\n\ninstall: clean prepare fmt vet gogenerate\n\tGOBIN=${PWD}/bin go install -ldflags \"-X main.version=${SCQL_VERSION}\" ./cmd/...\n\ngogenerate:\n\tgo generate ./pkg/...\n\tgo generate ./cmd/...\n\nfast: fmt vet\n\tGOBIN=${PWD}/bin go install ./cmd/...\n\nparser:\n\tcd pkg/parser && make\n\nbinary: clean prepare fmt vet gogenerate\n\t$(eval SCQL_VERSION := $(shell bash ${PWD}/version_build.sh))\n\techo \"Binary version: ${SCQL_VERSION}\"\n\tGOBIN=${PWD}/bin go install -ldflags \"-X main.version=${SCQL_VERSION}\" ./cmd/...\n\tbazelisk --host_jvm_args=-Xmx8g build //engine/exe:scqlengine -c opt --jobs=32\n\tbash ${PWD}/version_build.sh -r\n\nbinary-cov: clean prepare fmt vet gogenerate\n\t$(eval SCQL_VERSION := $(shell bash ${PWD}/version_build.sh))\n\techo \"Binary version: ${SCQL_VERSION}\"\n\tGOBIN=${PWD}/bin go install -ldflags \"-X main.version=${SCQL_VERSION}\" -cover ./...\n\tbazelisk --host_jvm_args=-Xmx8g build //engine/exe:scqlengine --jobs=32 --collect_code_coverage --instrumentation_filter=//engine/... --noremote_upload_local_results --copt=-DNDEBUG\n\tbash ${PWD}/version_build.sh -r\n\npb: clean\n\t$(RM) -rf pkg/proto-gen/*\n\t./api/generate_proto.sh\n\nfmt:\n\tgo fmt ./pkg/...\n\nvet:\n\tgo vet -unsafeptr=false ./pkg/...\n\ndoc:\n\tgo run ./cmd/docgen/main.go\n\tcd docs && rm -rf _build && make build\n\nlint: GOLINT-exists\n\t-${TOOLBIN}/golangci-lint run --out-format=colored-line-number\n\ndetect-shadowing:\n\tgo vet -vettool=$(shell which shadow) -strict ./...\n\nclean:\n\t$(RM) bin/*\n\t$(RM) *.coverprofile\n\ntest:\n\tgo test -v -cover ./pkg/...\n\ntestsum:\n\tgo run gotest.tools/gotestsum@latest ./pkg/...\n\ncoverage: install\n\tgo list -f '{{if gt (len .TestGoFiles) 0}}\"go test -covermode count -coverprofile {{.Name}}.coverprofile -coverpkg ./... {{.ImportPath}}\"{{end}}' ./... | xargs -I {} bash -c {}\n\tfind . -name \"*.coverprofile\"\n\t$(info Use `go tool cover -html MODULE_NAME.coverprofile`)\n\nprepare: GO-exists GO-package\n\nGO-exists:\n\t$(if $(shell command -v go 2> /dev/null),$(info Found `go`),$(error Please install go (prefer v1.22): refer to `https://golang.org/dl/`))\n\tgo version\n\tgo env GOPROXY\n\nGOLINT-exists:\n\t$(if $(shell command -v golangci-lint 2> /dev/null),$(info Found `golangci-lint`),$(shell curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b ${TOOLBIN} v1.61.0))\n\nGO-package:\n\t@GOBIN=${TOOLBIN} go install go.uber.org/mock/mockgen@latest && \\\n\tGOBIN=${TOOLBIN} go install golang.org/x/tools/cmd/goyacc@latest && \\\n\tGOBIN=${TOOLBIN} go install github.com/mattn/goveralls@latest && \\\n\tGOBIN=${TOOLBIN} go install github.com/rakyll/gotest@latest && \\\n\tGOBIN=${TOOLBIN} go install golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow@latest\n" }, { "path": "README.md", "content": "# SCQL\n\n[![CircleCI](https://dl.circleci.com/status-badge/img/gh/secretflow/scql/tree/main.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/secretflow/scql/tree/main)\n\nSecure Collaborative Query Language (SCQL) is a system that translates SQL statements into Secure Multiparty Computation (SMC) primitives and executes them on a federation of database systems.\n\n> **Note**: If you are looking for SCQL 1.0, please check the [1.x branch](https://github.com/secretflow/scql/tree/release/1.x).\n\n![SCQL Workflow](./docs/imgs/scql_workflow.png)\n\n## Documentation\n\n- [Documentation in English](https://www.secretflow.org.cn/en/docs/scql)\n- [中文文档](https://www.secretflow.org.cn/zh-CN/docs/scql/)\n\n## Docker Image Release\n\n- Official release docker image: [secretflow/scql](https://hub.docker.com/r/secretflow/scql/tags)\n- We also have images at Alibaba Cloud: secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/scql:[tag]\n\n## Contribution Guidelines\n\nIf you would like to contribute to SCQL, please see the [Contribution guidelines](CONTRIBUTING.md).\n\nThis documentation also contains instructions for [build and testing](CONTRIBUTING.md#build)\n\n## Hardware Requirements\n\nThe following requirements only apply to SCQLEngine.\n\n- CPU\n - x86_64: minimum required AVX instruction set. For FourQ based PSI, the AVX2 instruction set is required.\n\n## Disclaimer\n\nNon-release versions of SCQL are prohibited to use in any production environment due to possible bugs, glitches, lack of functionality, security issues or other problems.\n\n## Citing SCQL\n\nIf you think SCQL is helpful for your research or development, please consider citing our [paper](https://www.vldb.org/pvldb/vol17/p3987-fang.pdf):\n\n```text\n@article{scql,\n author = {Wenjing Fang and\n Shunde Cao and\n Guojin Hua and\n Junming Ma and\n Yongqiang Yu and\n Qunshan Huang and\n Jun Feng and\n Jin Tan and\n Xiaopeng Zan and\n Pu Duan and\n Yang Yang and\n Li Wang and\n Ke Zhang and\n Lei Wang},\n title = {SecretFlow-SCQL: {A} Secure Collaborative Query pLatform},\n journal = {Proc. VLDB Endow.},\n volume = {17},\n number = {12},\n pages = {3987--4000},\n year = {2024},\n url = {https://www.vldb.org/pvldb/vol17/p3987-fang.pdf},\n}\n```\n\n## Acknowledgments\n\n- Thanks [TiDB](https://github.com/pingcap/tidb) for providing a powerful SQL parser and planner.\n" }, { "path": "REPO_LAYOUT.md", "content": "# Repository Layout\n\n- [api/](api/): SCQL protocol files.\n- [examples/](examples/): SCQL examples.\n - [tutorial](examples/tutorial/): SCQL open core tutorial.\n- [docs/](docs/): Documents of SCQL.\n- [cmd/](cmd/): Main applications for SCQL.\n - [docgen/](cmd/docgen/): SCQL operators document generator.\n- [pkg/](pkg/): SCQL library code.\n - [constant/](pkg/constant/): Common constant values.\n - [parser/](pkg/parser/): SCQL parser.\n - [types/](pkg/types/): SCQL data types.\n - [table/](pkg/table/): SCQL table info.\n - [sessionctx/](pkg/sessionctx/): SCQL logical plan context.\n - [util/](pkg/util/): SCQL utils.\n - [planner/](pkg/planner/): SCQL planner.\n - [interpreter/](pkg/interpreter/): SCQL interpreter.\n - [compiler/](pkg/interpreter/compiler/): SCQL compiler that translates SQL to execution graph.\n - [graph/](pkg/interpreter/graph/): Execution graph data structures and processing.\n - [operator/](pkg/interpreter/operator/): SCQL operators.\n - [sc/](pkg/interpreter/sc/): SCQL compiler Python bindings using gopy.\n - [executor/](pkg/executor/): DQL executor. It dispatches execution dag to SCQL engine.\n- [engine/](engine/): SCQL execution engine, implemented in C++.\n - [exe/](engine/exe/): SCQL execution engine applications.\n - [services/](engine/services/): Engine RPC services.\n - [link/](engine/link/): MPI framework based on [YACL link](https://github.com/secretflow/yacl/tree/main/yacl/link).\n - [core/](engine/core/): Basic data structures used in engine.\n - [framework/](engine/framework/): Engine framework.\n - [operator/](engine/operator/): Oblivious operators.\n - [datasource/](engine/datasource/): SCQL data source adaptors/connectors.\n - [util/](engine/util/): Engine utilities.\n" }, { "path": "SECURITY.md", "content": "# Security\n\nIf you believe you have found a security vulnerability in any SecretFlow repository that meets\n[SecretFlow's definition of a security vulnerability](https://security.alipay.com/announcement.htm?id=1), please report it to us as described below.\n\n## Reporting Security Issues\n\n**Please do not report security vulnerabilities through public GitHub issues.**\n\nInstead, please report them to the ANT GROUP SECURITY Response Center at [https://security.alipay.com/](https://security.alipay.com/).\n\nIf you prefer to submit without logging in, send email to [antsrc@alipay.com](mailto:antsrc@alipay.com).\n\nYou should receive a response within 48 hours. If for some reason you do not, please follow up via email to ensure we received your original message.\nAdditional information can be found at [https://security.alipay.com/](https://security.alipay.com/).\n\nPlease include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:\n\n* Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)\n* Full paths of source file(s) related to the manifestation of the issue\n* The location of the affected source code (tag/branch/commit or direct URL)\n* Any special configuration required to reproduce the issue\n* Step-by-step instructions to reproduce the issue\n* Proof-of-concept or exploit code (if possible)\n* Impact of the issue, including how an attacker might exploit the issue\n\nThis information will help us triage your report more quickly.\n\n## Preferred Languages\n\nWe prefer all communications to be in Chinese or English.\n" }, { "path": "api/BUILD.bazel", "content": "# Copyright 2023 Ant Group Co., Ltd.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nload(\"@rules_buf//buf:defs.bzl\", \"buf_lint_test\")\nload(\"@rules_cc//cc:defs.bzl\", \"cc_proto_library\")\nload(\"@rules_go//proto:def.bzl\", \"go_proto_library\")\nload(\"@rules_java//java:defs.bzl\", \"java_proto_library\")\nload(\"@rules_proto//proto:defs.bzl\", \"proto_library\")\n\npackage(default_visibility = [\"//visibility:public\"])\n\nproto_library(\n name = \"core_proto\",\n srcs = [\"core.proto\"],\n deps = [\n \":common_proto\",\n \":status_proto\",\n ],\n)\n\nproto_library(\n name = \"status_code_proto\",\n srcs = [\"status_code.proto\"],\n)\n\nproto_library(\n name = \"engine_proto\",\n srcs = [\"engine.proto\"],\n deps = [\n \":common_proto\",\n \":core_proto\",\n \":status_proto\",\n \":subgraph_proto\",\n \"@com_google_protobuf//:empty_proto\",\n \"@spulib//libspu:spu_proto\",\n ],\n)\n\nproto_library(\n name = \"status_proto\",\n srcs = [\"status.proto\"],\n deps = [\n \"@com_google_protobuf//:any_proto\",\n ],\n)\n\nproto_library(\n name = \"common_proto\",\n srcs = [\"common.proto\"],\n deps = [\n \"@com_google_protobuf//:any_proto\",\n \"@com_google_protobuf//:timestamp_proto\",\n ],\n)\n\ncc_proto_library(\n name = \"common_cc_proto\",\n deps = [\":common_proto\"],\n)\n\ncc_proto_library(\n name = \"engine_cc_proto\",\n deps = [\":engine_proto\"],\n)\n\ncc_proto_library(\n name = \"core_cc_proto\",\n deps = [\":core_proto\"],\n)\n\ncc_proto_library(\n name = \"status_cc_proto\",\n deps = [\":status_code_proto\"],\n)\n\ncc_proto_library(\n name = \"interpreter_cc_proto\",\n deps = [\":interpreter_proto\"],\n)\n\nproto_library(\n name = \"subgraph_proto\",\n srcs = [\n \"subgraph.proto\",\n ],\n deps = [\":core_proto\"],\n)\n\nproto_library(\n name = \"interpreter_proto\",\n srcs = [\"interpreter.proto\"],\n deps = [\n \":common_proto\",\n \":core_proto\",\n \":status_proto\",\n \":subgraph_proto\",\n \"@com_google_protobuf//:timestamp_proto\",\n \"@spulib//libspu:spu_proto\",\n ],\n)\n\nproto_library(\n name = \"scql_task_proto\",\n srcs = [\"scql_task.proto\"],\n deps = [\n \":common_proto\",\n \":engine_proto\",\n \":subgraph_proto\",\n \"@spulib//libspu:spu_proto\",\n ],\n)\n\ncc_proto_library(\n name = \"scql_task_cc_proto\",\n deps = [\":scql_task_proto\"],\n)\n\ngo_proto_library(\n name = \"spu_go_proto\",\n importpath = \"github.com/secretflow/scql/pkg/proto-gen/spu\",\n protos = [\"@spulib//libspu:spu_proto\"],\n)\n\ngo_proto_library(\n name = \"googleapis_go_proto\",\n importpath = \"google.golang.org/genproto/googleapis/api\",\n protos = [\n \"@googleapis//google/api:annotations_proto\",\n \"@googleapis//google/api:field_behavior_proto\",\n \"@googleapis//google/api:http_proto\",\n ],\n)\n\ngo_proto_library(\n name = \"scql_go_proto\",\n compilers = [\"@rules_go//proto:go_grpc\"],\n importpath = \"github.com/secretflow/scql/pkg/proto-gen/scql\",\n protos = [\n \":common_proto\",\n \":core_proto\",\n \":engine_proto\",\n \":interpreter_proto\",\n \":scql_task_proto\",\n \":status_code_proto\",\n \":status_proto\",\n \":subgraph_proto\",\n ],\n deps = [\n \":googleapis_go_proto\",\n \":spu_go_proto\",\n ],\n)\n\n# lint\n\nbuf_lint_test(\n name = \"core_proto_lint\",\n config = \"buf.yaml\",\n targets = [\":core_proto\"],\n)\n\nbuf_lint_test(\n name = \"status_code_proto_lint\",\n config = \"buf.yaml\",\n targets = [\":status_code_proto\"],\n)\n\nbuf_lint_test(\n name = \"engine_proto_lint\",\n config = \"buf.yaml\",\n targets = [\":engine_proto\"],\n)\n\nbuf_lint_test(\n name = \"status_proto_lint\",\n config = \"buf.yaml\",\n targets = [\":status_proto\"],\n)\n\nbuf_lint_test(\n name = \"common_proto_lint\",\n config = \"buf.yaml\",\n targets = [\":common_proto\"],\n)\n\nbuf_lint_test(\n name = \"subgraph_proto_lint\",\n config = \"buf.yaml\",\n targets = [\":subgraph_proto\"],\n)\n\nbuf_lint_test(\n name = \"interpreter_proto_lint\",\n config = \"buf.yaml\",\n targets = [\":interpreter_proto\"],\n)\n" }, { "path": "api/buf.yaml", "content": "version: v2\nmodules:\n - path: .\n name: buf.build/secretflow/scql-protos\ndeps:\n - buf.build/googleapis/googleapis\n - buf.build/protocolbuffers/wellknowntypes\nlint:\n use:\n - DEFAULT\n - PACKAGE_DIRECTORY_MATCH\n - COMMENTS\n - UNARY_RPC\n except:\n # Allow direct resource returns for Create/Get operations following Google AIP patterns\n - RPC_RESPONSE_STANDARD_NAME\n - RPC_REQUEST_RESPONSE_UNIQUE\n ignore_only:\n # You can ignore specific files or rules if needed\n # - path/to/file.proto\nbreaking:\n use:\n - FILE\n" }, { "path": "api/common.proto", "content": "// Copyright 2023 Ant Group Co., Ltd.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//\n\nsyntax = \"proto3\";\n\npackage scql.pb;\n\nimport \"google/protobuf/any.proto\";\nimport \"google/protobuf/timestamp.proto\";\n\noption go_package = \"proto-gen/scql\";\noption java_package = \"org.secretflow.scql\";\n\n// RequestHeader carries the user custom headers.\nmessage RequestHeader {\n // Custom headers used to record custom information.\n map custom_headers = 1;\n}\n\nmessage DebugOptions {\n bool enable_psi_detail_log = 1;\n}\n\n// (-- TODO: move SQLWarning to a proper place --)\nmessage SQLWarning {\n // Description of the warning\n string reason = 1;\n}\n\n// IOStats contains input/output statistics for a job, including bytes sent and\n// received, as well as the number of send and receive actions performed.\nmessage IOStats {\n uint64 send_bytes = 1;\n uint64 recv_bytes = 2;\n uint64 send_actions = 3;\n uint64 recv_actions = 4;\n}\n\n// StageInfo provides information about an individual stage within a running\n// job.\nmessage StageInfo {\n // The name of the stage.\n string name = 1;\n // A brief summary describing the stage\n string summary = 2;\n // Stage start time\n google.protobuf.Timestamp start_time = 3;\n // Personalized details that may have different structures depending on the\n // stage type.\n google.protobuf.Any details = 4;\n}\n\n// JobProgress provides detailed information about the progress of a running\n// job.\nmessage JobProgress {\n // Job start time\n google.protobuf.Timestamp start_time = 1;\n // The total number of stages planned for the job.\n int32 stages_count = 2;\n // The number of stages that have been executed so far.\n int32 executed_stages = 3;\n IOStats io_stats = 4;\n // A list of currently running stages, providing insight into which parts of\n // the job are active.\n repeated StageInfo running_stages = 5;\n}\n\nenum JobState {\n JOB_STATE_UNSPECIFIED = 0;\n JOB_INITIALIZED = 1;\n JOB_RUNNING = 2;\n JOB_SUCCEEDED = 3;\n JOB_FAILED = 4;\n JOB_CANCELED = 5;\n}\n\nmessage LinkConfig {\n int64 link_recv_timeout_sec = 1;\n int64 link_throttle_window_size = 2;\n int64 link_chunked_send_parallel_size = 3;\n int64 http_max_payload_size = 4;\n}\n\nenum PsiAlgorithmType {\n // auto means choosing psi type by engine\n AUTO = 0;\n ECDH = 1;\n OPRF = 2;\n RR22 = 3;\n}\n\nenum Rr22Mode {\n UNDEFINED = 0;\n LOW_MODE = 1;\n FAST_MODE = 2;\n}\n\nmessage PsiConfig {\n int32 psi_curve_type = 1;\n PsiAlgorithmType psi_type = 2;\n // activated when psi_type is rr22\n Rr22Mode rr22_mode = 3;\n}\n\nmessage LogConfig {\n bool enable_session_logger_separation = 1;\n}\n\nmessage Placeholder {\n // placeholder name\n string name = 1;\n // placeholder name, e.g. string, int64, float\n string data_type = 2;\n}\n\nmessage Placeholders {\n repeated Placeholder placeholders = 1;\n}\n\nmessage Variable {\n string name = 1;\n string string_data = 2;\n int64 int64_data = 3;\n float float_data = 4;\n bool bool_data = 5;\n}\n" }, { "path": "api/core.proto", "content": "// Copyright 2023 Ant Group Co., Ltd.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//\n\nsyntax = \"proto3\";\n\npackage scql.pb;\n\nimport \"api/common.proto\";\nimport \"api/status.proto\";\n\noption go_package = \"proto-gen/scql\";\noption java_package = \"org.secretflow.scql\";\n\n// Defines a tensor shape. A dimension can be either an integer value\n// or a symbolic variable. A symbolic variable represents an unknown\n// dimension.\nmessage TensorShape {\n message Dimension {\n oneof value {\n int64 dim_value = 1;\n string dim_param = 2; // shape is unknown.\n }\n }\n repeated Dimension dim = 1;\n}\n\nenum PrimitiveDataType {\n PrimitiveDataType_UNDEFINED = 0;\n\n // Numeric types\n INT8 = 1; // the 8-bit signed integer type\n INT16 = 2; // the 16-bit signed integer type\n INT32 = 3; // the 32-bit signed integer type\n INT64 = 4; // the 64-bit signed integer type\n FLOAT32 = 5; // the 32-bit binary floating point type\n FLOAT64 = 6; // the 64-bit binary floating point type\n\n // Other types\n BOOL = 7;\n STRING = 8;\n // DATETIME and TIMESTAMP\n DATETIME = 9; // https://dev.mysql.com/doc/refman/8.0/en/datetime.html\n TIMESTAMP = 10; // seconds since '1970-01-01 00:00:00' UTC\n DECIMAL = 11;\n}\n\n// Tensor options.\nenum TensorOptions {\n // A tensor with data.\n VALUE = 0;\n // A tensor with reference (URI).\n REFERENCE = 1;\n // A tensor variable (declaration).\n VARIABLE = 2;\n}\n\nenum TensorStatus {\n // Unknown.\n TENSORSTATUS_UNKNOWN = 0;\n\n // Private.\n TENSORSTATUS_PRIVATE = 1;\n\n // Secret, usually in the form of secret sharing.\n TENSORSTATUS_SECRET = 2;\n\n // Ciphertext, usually in the form of homomorphic encryption ciphertext.\n TENSORSTATUS_CIPHER = 3;\n\n // Public.\n TENSORSTATUS_PUBLIC = 4;\n}\n\nmessage TensorAnnotation {\n TensorStatus status = 1;\n}\n\n// A tensor data representation.\nmessage Tensor {\n // Tensor name.\n string name = 1;\n\n // Tensor shape.\n // In SCQL cases, it's normally [M] (a vector with M elements).\n TensorShape shape = 2;\n\n // Tensor element type.\n PrimitiveDataType elem_type = 3;\n // used by decimal type\n int32 scale = 14;\n int32 width = 15;\n // Tensor options.\n TensorOptions option = 4;\n\n // Tensor annotation carries physical status information.\n // It MUST be there if the