Repository: enqueue/metabase-clickhouse-driver Branch: master Commit: c628179f468d Files: 52 Total size: 226.5 KB Directory structure: gitextract_tpemy9w4/ ├── .build/ │ └── Dockerfile ├── .docker/ │ ├── clickhouse/ │ │ ├── cluster/ │ │ │ ├── server1_config.xml │ │ │ ├── server1_macros.xml │ │ │ ├── server2_config.xml │ │ │ └── server2_macros.xml │ │ ├── single_node/ │ │ │ ├── config.xml │ │ │ └── users.xml │ │ ├── single_node_tls/ │ │ │ ├── Dockerfile │ │ │ ├── certificates/ │ │ │ │ ├── ca.crt │ │ │ │ ├── ca.key │ │ │ │ ├── client.crt │ │ │ │ ├── client.key │ │ │ │ ├── server.crt │ │ │ │ └── server.key │ │ │ ├── config.xml │ │ │ └── users.xml │ │ └── users.xml │ ├── nginx/ │ │ └── local.conf │ └── setup/ │ ├── Dockerfile │ ├── requirements.txt │ └── setup.py ├── .github/ │ ├── ISSUE_TEMPLATE/ │ │ ├── bug_report.md │ │ ├── feature_request.md │ │ └── question.md │ ├── deps.edn │ ├── pull_request_template.md │ └── workflows/ │ ├── check-jdbc-snapshot.yml │ └── check.yml ├── .gitignore ├── AUTHORS.md ├── CHANGELOG.md ├── CONTRIBUTING.md ├── HISTORY.md ├── LICENSE ├── README.md ├── build_docker_image.sh ├── deps.edn ├── docker-compose.yml ├── resources/ │ └── metabase-plugin.yaml ├── src/ │ └── metabase/ │ └── driver/ │ ├── clickhouse.clj │ ├── clickhouse_introspection.clj │ ├── clickhouse_nippy.clj │ ├── clickhouse_qp.clj │ └── clickhouse_version.clj └── test/ └── metabase/ ├── driver/ │ ├── clickhouse_data_types_test.clj │ ├── clickhouse_impersonation_test.clj │ ├── clickhouse_introspection_test.clj │ ├── clickhouse_substitution_test.clj │ ├── clickhouse_temporal_bucketing_test.clj │ └── clickhouse_test.clj └── test/ └── data/ ├── clickhouse.clj └── clickhouse_datasets.sql ================================================ FILE CONTENTS ================================================ ================================================ FILE: .build/Dockerfile ================================================ ARG METABASE_VERSION FROM metabase/metabase:$METABASE_VERSION COPY ./clickhouse.metabase-driver.jar /plugins/clickhouse.metabase-driver.jar RUN chmod 744 /plugins/clickhouse.metabase-driver.jar ================================================ FILE: .docker/clickhouse/cluster/server1_config.xml ================================================ 8123 9009 clickhouse1 users.xml default default 5368709120 /var/lib/clickhouse/ /var/lib/clickhouse/tmp/ /var/lib/clickhouse/user_files/ /var/lib/clickhouse/access/ 3 debug /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.err.log 1000M 10 1 clickhouse1 9000 clickhouse2 9000 9181 1 /var/lib/clickhouse/coordination/log /var/lib/clickhouse/coordination/snapshots 10000 30000 trace 10000 1 clickhouse1 9000 2 clickhouse2 9000 clickhouse1 9181 clickhouse2 9181 /clickhouse/test_cluster/task_queue/ddl system query_log
toYYYYMM(event_date) 1000 /var/lib/clickhouse/format_schemas/ users.xml ================================================ FILE: .docker/clickhouse/cluster/server1_macros.xml ================================================ test_cluster clickhouse1 1 ================================================ FILE: .docker/clickhouse/cluster/server2_config.xml ================================================ 8123 9009 clickhouse2 users.xml default default 5368709120 /var/lib/clickhouse/ /var/lib/clickhouse/tmp/ /var/lib/clickhouse/user_files/ /var/lib/clickhouse/access/ 3 debug /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.err.log 1000M 10 1 clickhouse1 9000 clickhouse2 9000 9181 2 /var/lib/clickhouse/coordination/log /var/lib/clickhouse/coordination/snapshots 10000 30000 trace 10000 1 clickhouse1 9000 2 clickhouse2 9000 clickhouse1 9181 clickhouse2 9181 /clickhouse/test_cluster/task_queue/ddl system query_log
toYYYYMM(event_date) 1000 /var/lib/clickhouse/format_schemas/ users.xml ================================================ FILE: .docker/clickhouse/cluster/server2_macros.xml ================================================ test_cluster clickhouse2 1 ================================================ FILE: .docker/clickhouse/single_node/config.xml ================================================ 8123 9000 users.xml default default 5368709120 /var/lib/clickhouse/ /var/lib/clickhouse/tmp/ /var/lib/clickhouse/user_files/ /var/lib/clickhouse/access/ UTC debug /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.err.log 1000M 10 1 system query_log
toYYYYMM(event_date) 1000 /var/lib/clickhouse/format_schemas/ users.xml ================================================ FILE: .docker/clickhouse/single_node/users.xml ================================================ random 1 ::/0 default default 1 foo@bar! ::/0 default default 1 3600 0 0 0 0 0 ================================================ FILE: .docker/clickhouse/single_node_tls/Dockerfile ================================================ FROM clickhouse/clickhouse-server:25.2-alpine COPY .docker/clickhouse/single_node_tls/certificates /etc/clickhouse-server/certs RUN chown clickhouse:clickhouse -R /etc/clickhouse-server/certs \ && chmod 600 /etc/clickhouse-server/certs/* \ && chmod 755 /etc/clickhouse-server/certs ================================================ FILE: .docker/clickhouse/single_node_tls/certificates/ca.crt ================================================ -----BEGIN CERTIFICATE----- MIICTTCCAdKgAwIBAgIUaqbLNiwUtbV5VuolTMGXOO+21vEwCgYIKoZIzj0EAwQw XTELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMSAwHgYDVQQKDBdDbGlja0hvdXNl IENvbm5lY3QgVGVzdDEfMB0GA1UEAwwWY2xpY2tob3VzZWNvbm5lY3QudGVzdDAe Fw0yMjA1MTkxODIxMzFaFw00MjA1MTQxODIxMzFaMF0xCzAJBgNVBAYTAlVTMQsw CQYDVQQIDAJDQTEgMB4GA1UECgwXQ2xpY2tIb3VzZSBDb25uZWN0IFRlc3QxHzAd BgNVBAMMFmNsaWNraG91c2Vjb25uZWN0LnRlc3QwdjAQBgcqhkjOPQIBBgUrgQQA IgNiAATTKvPxkWILniWZ9EmcftQRqhH7fpVhQm1hvtZW1cpTozV0z6tdopnS5p/W l+Kti2k/kZx1rsN1ZrRYKJN8ANruJJ6vaDOjbf89cmViZ/dbOi49T8brTzdHeuGI E2TyP+WjUzBRMB0GA1UdDgQWBBThZgdf9aToyK2TeSQ+suyjNUuifDAfBgNVHSME GDAWgBThZgdf9aToyK2TeSQ+suyjNUuifDAPBgNVHRMBAf8EBTADAQH/MAoGCCqG SM49BAMEA2kAMGYCMQDWQUTb39xLLds0WobJmNQbIkEwZyss0XNQkn6qI8rz73NL 6L5/6wNzetKhBf3WBCYCMQC+evVR3Td+WLfbKQDXrCbSkogW6++I/9l55wakMz9G P+0she/nvFuUKnB+VRcaBqM= -----END CERTIFICATE----- ================================================ FILE: .docker/clickhouse/single_node_tls/certificates/ca.key ================================================ -----BEGIN EC PARAMETERS----- BgUrgQQAIg== -----END EC PARAMETERS----- -----BEGIN EC PRIVATE KEY----- MIGkAgEBBDA26CfROCd9PBA7jUKmImXeYSuExnZcdNeX6AR0Aaj1wgZ/iH3hoQYH bPi7JnRTo+6gBwYFK4EEACKhZANiAATTKvPxkWILniWZ9EmcftQRqhH7fpVhQm1h vtZW1cpTozV0z6tdopnS5p/Wl+Kti2k/kZx1rsN1ZrRYKJN8ANruJJ6vaDOjbf89 cmViZ/dbOi49T8brTzdHeuGIE2TyP+U= -----END EC PRIVATE KEY----- ================================================ FILE: .docker/clickhouse/single_node_tls/certificates/client.crt ================================================ -----BEGIN CERTIFICATE----- MIIB+TCCAX8CFEc86vC0vsMjLzQzxazHeHjQblL2MAoGCCqGSM49BAMEMF0xCzAJ BgNVBAYTAlVTMQswCQYDVQQIDAJDQTEgMB4GA1UECgwXQ2xpY2tIb3VzZSBDb25u ZWN0IFRlc3QxHzAdBgNVBAMMFmNsaWNraG91c2Vjb25uZWN0LnRlc3QwHhcNMjIw NTE5MjEwNTA2WhcNNDIwNTEzMjEwNTA2WjBkMQswCQYDVQQGEwJVUzELMAkGA1UE CAwCQ0ExIDAeBgNVBAoMF0NsaWNrSG91c2UgQ29ubmVjdCBUZXN0MSYwJAYDVQQD DB1jbGllbnQuY2xpY2tob3VzZWNvbm5lY3QudGVzdDB2MBAGByqGSM49AgEGBSuB BAAiA2IABBrSSv+9xHsp8Bge3wdoO+3VdDM4DDrocE0Gm+EW65MN6/6oDmbyKOB1 JbTY0aq3lIN9PtUibCrGDqcVqtQnihnvTIDLqK0Xlxvv6Jc0t6DvXYaKhg6jIimt B7NEvysGVzAKBggqhkjOPQQDBANoADBlAjBblevbpaRlekX7fH16KnYttGoIqDBI 45LlBJ2sEe5qSKCBoLdN89Tk8WD4lG7PhlkCMQDdFd8OKMPaZiUWIdHI6AeDWwXD bJi0LwDxXgyBVCGLZ2vTbOVxnr2Qp+9BjFURU8c= -----END CERTIFICATE----- ================================================ FILE: .docker/clickhouse/single_node_tls/certificates/client.key ================================================ -----BEGIN EC PARAMETERS----- BgUrgQQAIg== -----END EC PARAMETERS----- -----BEGIN EC PRIVATE KEY----- MIGkAgEBBDCsmtNREbSJvGTTIQmx019frOllv9m2EqOWesvqu52rH6tIxdw5TE4A SZICvNKYzH2gBwYFK4EEACKhZANiAAQa0kr/vcR7KfAYHt8HaDvt1XQzOAw66HBN BpvhFuuTDev+qA5m8ijgdSW02NGqt5SDfT7VImwqxg6nFarUJ4oZ70yAy6itF5cb 7+iXNLeg712GioYOoyIprQezRL8rBlc= -----END EC PRIVATE KEY----- ================================================ FILE: .docker/clickhouse/single_node_tls/certificates/server.crt ================================================ -----BEGIN CERTIFICATE----- MIIDPTCCAsOgAwIBAgIURzzq8LS+wyMvNDPFrMd4eNBuUvUwCgYIKoZIzj0EAwQw XTELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMSAwHgYDVQQKDBdDbGlja0hvdXNl IENvbm5lY3QgVGVzdDEfMB0GA1UEAwwWY2xpY2tob3VzZWNvbm5lY3QudGVzdDAe Fw0yMjA1MTkyMDU3MjRaFw00MjA1MTMyMDU3MjRaMGQxCzAJBgNVBAYTAlVTMQsw CQYDVQQIDAJDQTEgMB4GA1UECgwXQ2xpY2tIb3VzZSBDb25uZWN0IFRlc3QxJjAk BgNVBAMMHXNlcnZlci5jbGlja2hvdXNlY29ubmVjdC50ZXN0MHYwEAYHKoZIzj0C AQYFK4EEACIDYgAECsvHRYxPr+kJ/A7DDajEu8PhdO+WGxzJs7k9SdypPWSxOaCD ME2tWq0t0Giy63JYNhsn+CJglNIXhtfS5nHS7NV5SfBABUVtZS2/MFk8CwFCz+Rc Z4db2gt937AgjfxCo4IBOzCCATcwCQYDVR0TBAIwADARBglghkgBhvhCAQEEBAMC BkAwOQYJYIZIAYb4QgENBCwWKkNsaWNrSG91c2UgQ29ubmVjdCBUZXN0IFNlcnZl ciBDZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQUZDd2tpXw4FMDFcY38eXCb+tmukAwgZoG A1UdIwSBkjCBj4AU4WYHX/Wk6Mitk3kkPrLsozVLonyhYaRfMF0xCzAJBgNVBAYT AlVTMQswCQYDVQQIDAJDQTEgMB4GA1UECgwXQ2xpY2tIb3VzZSBDb25uZWN0IFRl c3QxHzAdBgNVBAMMFmNsaWNraG91c2Vjb25uZWN0LnRlc3SCFGqmyzYsFLW1eVbq JUzBlzjvttbxMAsGA1UdDwQEAwIF4DATBgNVHSUEDDAKBggrBgEFBQcDATAKBggq hkjOPQQDBANoADBlAjBc3W/8qr04xmUiDOHSEoug89cK8YxtRiKdCjiR3Lao1h5a J5Xc0JhVLaDUFb+blkoCMQCM7rKbO3itBKaweeJijX/veBcISYFulryWeANiltxo DFDHrC54rGXt4eOMouTlPbw= -----END CERTIFICATE----- ================================================ FILE: .docker/clickhouse/single_node_tls/certificates/server.key ================================================ -----BEGIN EC PARAMETERS----- BgUrgQQAIg== -----END EC PARAMETERS----- -----BEGIN EC PRIVATE KEY----- MIGkAgEBBDDyhVuBtGuyKEDr2HciKi4yS2T0WloMeUG2kgQRKim7Mih7977q7RbI t6sGwlsBxKGgBwYFK4EEACKhZANiAAQKy8dFjE+v6Qn8DsMNqMS7w+F075YbHMmz uT1J3Kk9ZLE5oIMwTa1arS3QaLLrclg2Gyf4ImCU0heG19LmcdLs1XlJ8EAFRW1l Lb8wWTwLAULP5Fxnh1vaC33fsCCN/EI= -----END EC PRIVATE KEY----- ================================================ FILE: .docker/clickhouse/single_node_tls/config.xml ================================================ 8443 9440 0.0.0.0 users.xml default default 5368709120 /var/lib/clickhouse/ /var/lib/clickhouse/tmp/ /var/lib/clickhouse/user_files/ /var/lib/clickhouse/access/ debug /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.err.log 1000M 10 1 /etc/clickhouse-server/certs/server.crt /etc/clickhouse-server/certs/server.key relaxed /etc/clickhouse-server/certs/ca.crt true sslv2,sslv3,tlsv1 true system query_log
toYYYYMM(event_date) 1000 /var/lib/clickhouse/format_schemas/ users.xml ================================================ FILE: .docker/clickhouse/single_node_tls/users.xml ================================================ random ::/0 default default 1 client.clickhouseconnect.test default 3600 0 0 0 0 0 ================================================ FILE: .docker/clickhouse/users.xml ================================================ random 1 ::/0 default default 1 foo@bar! ::/0 default default 1 3600 0 0 0 0 0 ================================================ FILE: .docker/nginx/local.conf ================================================ upstream clickhouse_cluster { server clickhouse1:8123; server clickhouse2:8123; } server { listen 8123; client_max_body_size 100M; location / { proxy_pass http://clickhouse_cluster; } } ================================================ FILE: .docker/setup/Dockerfile ================================================ FROM python:3.11.9-alpine COPY . /app/ RUN pip install -r /app/requirements.txt ================================================ FILE: .docker/setup/requirements.txt ================================================ requests==2.32.2 ================================================ FILE: .docker/setup/setup.py ================================================ import copy import logging import os import pprint import requests host = os.environ.get('host') if os.environ.get('host') else 'http://localhost' port = os.environ.get('port') if os.environ.get('port') else '3000' admin_email = os.environ.get('admin_email') if os.environ.get('admin_email') else 'admin@example.com' user_email = os.environ.get('user_email') if os.environ.get('user_email') else 'user@example.com' password = os.environ.get('password') if os.environ.get('password') else 'metabot1' site_name = 'ClickHouse test' endpoints = { 'health_check': '/api/health', 'properties': '/api/session/properties', 'setup': '/api/setup', 'database': '/api/database', 'login': '/api/session', 'user': '/api/user', } for k, v in endpoints.items(): endpoints[k] = f"{host}:{port}{v}" db_base_payload = { "is_on_demand": False, "is_full_sync": True, "is_sample": False, "cache_ttl": None, "refingerprint": False, "auto_run_queries": True, "schedules": {}, "details": { "host": "clickhouse", "port": 8123, "user": "default", "password": None, "dbname": "default", "scan-all-databases": False, "ssl": False, "tunnel-enabled": False, "advanced-options": False }, "name": "Our ClickHouse", "engine": "clickhouse" } def health(): response = requests.get(endpoints['health_check'], verify=False) if response.json()['status'] == 'ok': return 'healthy' else: health() def check_response(response, op): if response.status_code >= 300: print(f'Unexpected status {response.status_code} for {op}', response.text) exit(1) if __name__ == '__main__': print("Checking health") if health() == 'healthy' and os.environ.get('retry') is None: print("Healthy, setting up Metabase") session = requests.Session() session_token = None try: token = session.get(endpoints['properties'], verify=False).json()['setup-token'] setup_payload = { 'token': f'{token}', 'user': { 'first_name': 'Admin', 'last_name': 'Admin', 'email': admin_email, 'site_name': site_name, 'password': password, 'password_confirm': password }, 'database': None, 'invite': None, 'prefs': { 'site_name': site_name, 'site_locale': 'en', 'allow_tracking': False } } print("Getting the setup token") session_token = session.post(endpoints['setup'], verify=False, json=setup_payload).json()['id'] except Exception as e: print("The admin user was already created") try: if session_token is None: session_token = session.post(endpoints['login'], verify=False, json={"username": admin_email, "password": password}) dbs = session.get(endpoints['database'], verify=False).json() print("Current databases:") pprint.pprint(dbs['data']) sample_db = next((x for x in dbs['data'] if x['id'] == 1), None) if sample_db is not None: print("Deleting the sample database") res = session.delete(f"{endpoints['database']}/{sample_db['id']}") check_response(res, 'delete sample db') else: print("The sample database was already deleted") single_node_db = next((x for x in dbs['data'] if x['engine'] == 'clickhouse' and x['details']['host'] == 'clickhouse'), None) if single_node_db is None: print("Creating ClickHouse single node db") single_node_payload = copy.deepcopy(db_base_payload) single_node_payload['name'] = 'ClickHouse (single node)' res = session.post(endpoints['database'], verify=False, json=single_node_payload) check_response(res, 'create single node db') else: print("The single node database was already created") # cluster_db = next((x for x in dbs['data'] # if x['engine'] == 'clickhouse' # and x['details']['host'] == 'nginx'), None) # if cluster_db is None: # print("Creating ClickHouse cluster db") # cluster_db_payload = copy.deepcopy(db_base_payload) # cluster_db_payload['details']['host'] = 'nginx' # cluster_db_payload['name'] = 'ClickHouse (cluster)' # res = session.post(endpoints['database'], verify=False, json=cluster_db_payload) # check_response(res) # else: # print("The cluster database was already created") print("Creating a regular user") user_payload = {"first_name": "User", "last_name": "User", "email": user_email, "password": password} res = session.post(endpoints['user'], verify=False, json=user_payload) check_response(res, 'create user') print("Done!") except Exception as e: logging.exception("Failed to setup Metabase", e) exit() ================================================ FILE: .github/ISSUE_TEMPLATE/bug_report.md ================================================ --- name: Bug report about: Create a report to help us improve title: '' labels: bug assignees: '' --- ### Describe the bug ### Steps to reproduce 1. 2. 3. ### Expected behaviour ### Error log ### Configuration #### Environment * metabase-clickhouse-driver version: * metabase-clickhouse-driver configuration: * Metabase version: * OS: #### ClickHouse server * ClickHouse Server version: * ClickHouse Server non-default settings, if any: * `CREATE TABLE` statements for tables involved: * Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary ================================================ FILE: .github/ISSUE_TEMPLATE/feature_request.md ================================================ --- name: Feature request about: Suggest an idea for the driver title: '' labels: enhancement assignees: '' --- ### Use case ### Describe the solution you'd like ### Describe the alternatives you've considered ### Additional context ================================================ FILE: .github/ISSUE_TEMPLATE/question.md ================================================ --- name: Question about: Ask a question about the driver title: '' labels: question assignees: '' --- > Make sure to check the [documentation](https://clickhouse.com/docs/en/integrations/metabase) first. > If the question is concise and probably has a short answer, > asking it in the [community Slack](https://clickhouse.com/slack) is probably the fastest way to find the answer. ================================================ FILE: .github/deps.edn ================================================ {:aliases {:user/clickhouse {:extra-paths ["PWD/modules/drivers/clickhouse/test"] :extra-deps {metabase/clickhouse {:local/root "PWD/modules/drivers/clickhouse"}}}}} ================================================ FILE: .github/pull_request_template.md ================================================ ## Summary ## Checklist Delete items not relevant to your PR: - [ ] Unit and integration tests covering the common scenarios were added - [ ] A human-readable description of the changes was provided to include in CHANGELOG - [ ] For significant changes, documentation in https://github.com/ClickHouse/clickhouse-docs was updated with further explanations or tutorials ================================================ FILE: .github/workflows/check-jdbc-snapshot.yml ================================================ # sed -i -E "s/(com.clickhouse\/clickhouse-jdbc \{:mvn\/version )\".+?\"\}/\1\"\"}/" deps.edn name: Check with the latest JDBC snapshot on: workflow_dispatch: inputs: jdbc-version: description: "JDBC version to use" required: true env: # Temporarily using a fork to disable a few failing tests METABASE_REPOSITORY: slvrtrn/metabase METABASE_VERSION: 0.53.x-ch jobs: check-local-current-version: runs-on: ubuntu-latest steps: - name: Checkout Metabase Repo uses: actions/checkout@v4 with: repository: ${{ env.METABASE_REPOSITORY }} ref: ${{ env.METABASE_VERSION }} - name: Checkout Driver Repo uses: actions/checkout@v4 with: path: modules/drivers/clickhouse - name: Prepare JDK 21 uses: actions/setup-java@v4 with: distribution: "temurin" java-version: "21" - name: Add ClickHouse TLS instance to /etc/hosts run: | sudo echo "127.0.0.1 server.clickhouseconnect.test" | sudo tee -a /etc/hosts - name: Start ClickHouse in Docker uses: hoverkraft-tech/compose-action@v2.0.0 with: compose-file: "modules/drivers/clickhouse/docker-compose.yml" down-flags: "--volumes" services: | clickhouse clickhouse_tls clickhouse_older_version clickhouse_cluster_node1 clickhouse_cluster_node2 nginx - name: Install Clojure CLI run: | curl -O https://download.clojure.org/install/linux-install-1.11.1.1182.sh && sudo bash ./linux-install-1.11.1.1182.sh - name: Setup Node uses: actions/setup-node@v4 with: node-version: "20" cache: "yarn" - name: Update the JDBC version in deps.edn working-directory: modules/drivers/clickhouse run: | sed -i -E "s/(com.clickhouse\/clickhouse-jdbc \{:mvn\/version )\".+?\"\}/\1\"${{ github.event.inputs.jdbc-version }}\"}/" deps.edn cat deps.edn - name: Prepare stuff for pulses run: yarn build-static-viz # Use custom deps.edn containing "user/clickhouse" alias to include driver sources - name: Prepare deps.edn run: | mkdir -p /home/runner/.config/clojure cat modules/drivers/clickhouse/.github/deps.edn | sed -e "s|PWD|$PWD|g" > /home/runner/.config/clojure/deps.edn - name: Run all tests with the latest ClickHouse version env: DRIVERS: clickhouse run: | clojure -X:ci:dev:ee:ee-dev:drivers:drivers-dev:test:user/clickhouse ================================================ FILE: .github/workflows/check.yml ================================================ name: Check on: workflow_dispatch: push: branches: - master paths-ignore: - '**/*.md' pull_request: paths-ignore: - '**/*.md' env: # Using a fork to disable a few failing tests METABASE_REPOSITORY: slvrtrn/metabase METABASE_VERSION: 0.53.x-ch jobs: check-local-current-version: runs-on: ubuntu-latest steps: - name: Checkout Metabase Repo uses: actions/checkout@v4 with: repository: ${{ env.METABASE_REPOSITORY }} ref: ${{ env.METABASE_VERSION }} - name: Checkout Driver Repo uses: actions/checkout@v4 with: path: modules/drivers/clickhouse - name: Prepare JDK 21 uses: actions/setup-java@v4 with: distribution: "temurin" java-version: "21" - name: Add ClickHouse TLS instance to /etc/hosts run: | sudo echo "127.0.0.1 server.clickhouseconnect.test" | sudo tee -a /etc/hosts - name: Start ClickHouse in Docker uses: hoverkraft-tech/compose-action@v2.0.0 with: compose-file: "modules/drivers/clickhouse/docker-compose.yml" down-flags: "--volumes" services: | clickhouse clickhouse_tls clickhouse_older_version clickhouse_cluster_node1 clickhouse_cluster_node2 nginx - name: Install Clojure CLI run: | curl -O https://download.clojure.org/install/linux-install-1.11.1.1182.sh && sudo bash ./linux-install-1.11.1.1182.sh - name: Setup Node uses: actions/setup-node@v4 with: node-version: "20" cache: "yarn" - name: Get M2 cache uses: actions/cache@v4 with: path: | ~/.m2 ~/.gitlibs key: ${{ runner.os }}-clickhouse-${{ hashFiles('**/deps.edn') }} - name: Prepare stuff for pulses run: yarn build-static-viz # Use custom deps.edn containing "user/clickhouse" alias to include driver sources - name: Prepare deps.edn run: | mkdir -p /home/runner/.config/clojure cat modules/drivers/clickhouse/.github/deps.edn | sed -e "s|PWD|$PWD|g" > /home/runner/.config/clojure/deps.edn - name: Run all tests with the latest ClickHouse version env: DRIVERS: clickhouse run: | clojure -X:ci:dev:ee:ee-dev:drivers:drivers-dev:test:user/clickhouse check-local-older-version: runs-on: ubuntu-latest steps: - name: Checkout Metabase Repo uses: actions/checkout@v4 with: repository: ${{ env.METABASE_REPOSITORY }} ref: ${{ env.METABASE_VERSION }} - name: Checkout Driver Repo uses: actions/checkout@v4 with: path: modules/drivers/clickhouse - name: Prepare JDK 21 uses: actions/setup-java@v4 with: distribution: "temurin" java-version: "21" - name: Add ClickHouse TLS instance to /etc/hosts run: | sudo echo "127.0.0.1 server.clickhouseconnect.test" | sudo tee -a /etc/hosts - name: Start ClickHouse in Docker uses: hoverkraft-tech/compose-action@v2.0.0 with: compose-file: "modules/drivers/clickhouse/docker-compose.yml" down-flags: "--volumes" services: | clickhouse clickhouse_tls clickhouse_older_version clickhouse_cluster_node1 clickhouse_cluster_node2 nginx - name: Install Clojure CLI run: | curl -O https://download.clojure.org/install/linux-install-1.11.1.1182.sh && sudo bash ./linux-install-1.11.1.1182.sh - name: Get M2 cache uses: actions/cache@v4 with: path: | ~/.m2 ~/.gitlibs key: ${{ runner.os }}-clickhouse-${{ hashFiles('**/deps.edn') }} # Use custom deps.edn containing "user/clickhouse" alias to include driver sources - name: Prepare deps.edn run: | mkdir -p /home/runner/.config/clojure cat modules/drivers/clickhouse/.github/deps.edn | sed -e "s|PWD|$PWD|g" > /home/runner/.config/clojure/deps.edn - name: Run ClickHouse driver tests with 23.3 env: DRIVERS: clickhouse MB_CLICKHOUSE_TEST_PORT: 8124 run: | clojure -X:ci:dev:ee:ee-dev:drivers:drivers-dev:test:user/clickhouse :only metabase.driver.clickhouse-test build-jar: runs-on: ubuntu-latest needs: [ 'check-local-current-version' ] steps: - name: Checkout Metabase Repo uses: actions/checkout@v4 with: repository: ${{ env.METABASE_REPOSITORY }} ref: ${{ env.METABASE_VERSION }} - name: Checkout Driver Repo uses: actions/checkout@v4 with: path: modules/drivers/clickhouse - name: Install Clojure CLI run: | curl -O https://download.clojure.org/install/linux-install-1.11.1.1182.sh && sudo bash ./linux-install-1.11.1.1182.sh - name: Build ClickHouse driver run: | echo "{:deps {metabase/clickhouse {:local/root \"clickhouse\" }}}" > modules/drivers/deps.edn bin/build-driver.sh clickhouse ls -lah resources/modules - name: Archive driver JAR uses: actions/upload-artifact@v4 with: name: clickhouse.metabase-driver.jar path: resources/modules/clickhouse.metabase-driver.jar ================================================ FILE: .gitignore ================================================ \#*\# .\#* /target *.jar *.class /.lein-env /.lein-failures /.lein-plugins /.lein-repl-history /.clj-kondo .eastwood .calva .cpcache .joyride .nrepl-port .idea ================================================ FILE: AUTHORS.md ================================================ # Contributors, Authors The initial source base comprises major contributions from these authors (_the git log has suffered from frequent brutal rebases, please add yourself here, if I missed you!_): * Bogdan Mukvich (@Badya) * @tsl-karlp * Andrew Grigorev (@ei-grad) * Felix Mueller (@enqueue) ================================================ FILE: CHANGELOG.md ================================================ # 1.53.4 ### Improvements * The JDBC driver was updated to [0.8.4](https://github.com/ClickHouse/clickhouse-java/releases/tag/v0.8.4). This fixes [#309](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/309), [#300](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/300), [#297](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/297). # 1.53.3 ### Improvements * If ClickHouse instance hostname was specified including `http://` or `https://` schema (e.g. `https://sub.example.com`), it will be automatically handled and removed by the driver, instead of failing with a connection error. * The JDBC driver was updated to [0.8.2](https://github.com/ClickHouse/clickhouse-java/releases/tag/v0.8.2) # 1.53.2 ### Bug fixes * The JDBC driver was updated to [0.8.1](https://github.com/ClickHouse/clickhouse-java/releases/tag/v0.8.1) to fix errors in queries with CTEs ([#297](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/297), [#288](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/288), [tadeboro](https://github.com/tadeboro)). # 1.53.1 ### Bug fixes * Fix unsigned integers overflow ([#293](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/293)) # 1.53.0 ### Improvements * Adds Metabase 0.53.x support ([#287](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/287), [dpsutton](https://github.com/dpsutton)). ### Bug fixes * Fixed OOB exception on CSV insert caused by an incompatibility with JDBC v2 ([#286](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/286), [wotbrew](https://github.com/wotbrew)). # 1.52.0 - Formal Metabase 0.52.x+ support - The driver now uses JDBC v2 ([0.8.0](https://github.com/ClickHouse/clickhouse-java/releases/tag/v0.8.0)) - Various improvements to handling of datetimes with timezones - `:convert-timezone` feature is disabled for now. - Added `max-open-connections` setting under "advanced options"; default is 100. # 1.51.0 Adds Metabase 0.51.x support. # 1.50.7 ### Improvements * Added a configuration field (under the "advanced options", hidden by default) to override certain ClickHouse settings ([#272](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/272)). # 1.50.6 ### Bug fixes * Fixed null pointer exception on CSV insert ([#268](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/268), [crisptrutski](https://github.com/crisptrutski)). # 1.50.5 ### Bug fixes * Fixed an error that could occur while setting roles containing hyphens for connection impersonation ([#266](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/266), [sharankow](https://github.com/sharankow)). # 1.50.4 ### Bug fixes * Fixed an error while uploading a CSV with an offset datetime column ([#263](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/263), [crisptrutski](https://github.com/crisptrutski)). # 1.50.3 ### Improvements * The driver no longer explicitly sets `allow_experimental_analyzer=0` settings on the connection level; the [new ClickHouse analyzer](https://clickhouse.com/docs/en/operations/analyzer) is now enabled by default. # 1.50.2 ### Bug fixes * Fixed Array inner type introspection, which could cause reduced performance when querying tables containing arrays. ([#257](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/257)) # 1.50.1 ### New features * Enabled `:set-timezone` ([docs](https://www.metabase.com/docs/latest/configuring-metabase/localization#report-timezone)) Metabase feature in the driver. ([#200](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/200)) * Enabled `:convert-timezone` ([docs](https://www.metabase.com/docs/latest/questions/query-builder/expressions/converttimezone)) Metabase feature in the driver. ([#254](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/254)) ### Other * The driver now uses [`session_timezone` ClickHouse setting](https://clickhouse.com/docs/en/operations/settings/settings#session_timezone). This is necessary to support the `:set-timezone` and `:convert-timezone` features; however, this setting [was introduced in 23.6](https://clickhouse.com/docs/en/whats-new/changelog/2023#236), which makes it the minimal required ClickHouse version to work with the driver. # 1.50.0 After Metabase 0.50.0, a new naming convention exists for the driver's releases. The new one is intended to reflect the Metabase version the driver is supposed to run on. For example, the driver version 1.50.0 means that it should be used with Metabase v0.50.x or Metabase EE 1.50.x _only_, and it is _not guaranteed_ that this particular version of the driver can work with the previous or the following versions of Metabase. ### New features * Added Metabase 0.50.x support. ### Improvements * Bumped the JDBC driver to [0.6.1](https://github.com/ClickHouse/clickhouse-java/releases/tag/v0.6.1). ### Bug fixes * Fixed the issue where the connection impersonation feature support could be incorrectly reported as disabled. ### Other * The new ClickHouse analyzer, [which is enabled by default in 24.3+](https://clickhouse.com/blog/clickhouse-release-24-03#analyzer-enabled-by-default), is disabled for the queries executed by the driver, as it shows some compatibilities with the queries generated by Metabase (see [this issue](https://github.com/ClickHouse/ClickHouse/issues/64487) for more details). * The `:window-functions/offset` Metabase feature is currently disabled, as the default implementation generates queries incompatible with ClickHouse. See [this issue](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/245) for tracking. # 1.5.1 Metabase 0.49.14+ only. ### Bug fixes * Fixed the issue where the Metabase instance could end up broken if the ClickHouse instance was _stopped_ during the upgrade to 1.5.0. ([#242](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/242)) * Fixed variables substitution with Nullable Date, Date32, DateTime, and DateTime64 columns, where the generated query could fail with NULL values in the database due to an incorrect cast call. ([#243](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/243)) # 1.5.0 Metabase 0.49.14+ only. ### New features * Added [Metabase CSV Uploads feature](https://www.metabase.com/docs/latest/databases/uploads) support, which is currently enabled with ClickHouse Cloud only. On-premise deployments support will be added in the next release. ([calherries](https://github.com/calherries), [#236](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/236), [#238](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/238)) * Added [Metabase connection impersonation feature](https://www.metabase.com/learn/permissions/impersonation) support. This feature will be enabled by the driver only if ClickHouse version 24.4+ is detected. ([#219](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/219)) ### Improvements * Proper role setting support on cluster deployments (related issue: [#192](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/192)). * Bump the JDBC driver to [0.6.0-patch5](https://github.com/ClickHouse/clickhouse-java/releases/tag/v0.6.0-patch5). ### Bug fixes * Fixed missing data for the last day when using filters with DateTime columns. ([#202](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/202), [#229](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/229)) # 1.4.0 ### New features * Metabase 0.49.x support. ### Bug fixes * Fixed an incorrect substitution for the current day filter with DateTime columns. ([#216](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/216)) # 1.3.4 ### New features * If introspected ClickHouse version is lower than 23.8, the driver will not use [startsWithUTF8](https://clickhouse.com/docs/en/sql-reference/functions/string-functions#startswithutf8) and fall back to its [non-UTF8 counterpart](https://clickhouse.com/docs/en/sql-reference/functions/string-functions#startswith) instead. There is a drawback in this compatibility mode: potentially incorrect filtering results when working with non-latin strings. If your use case includes filtering by columns with such strings and you experience these issues, consider upgrading your ClickHouse server to 23.8+. ([#224](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/224)) # 1.3.3 ### Bug fixes * Fixed an issue where it was not possible to create a connection with multiple databases using TLS. ([#215](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/215)) # 1.3.2 ### Bug fixes * Remove `can-connect?` method override which could cause issues with editing or creating new connections. ([#212](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/212)) # 1.3.1 ### Bug fixes * Fixed incorrect serialization of `Array(UInt8)` columns ([#209](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/209)) # 1.3.0 ### New features * Metabase 0.48.x support ### Bug fixes * Fixed last/next minutes/hours filters with variables creating incorrect queries due to unnecessary `CAST col AS date` call. # 1.2.5 Metabase 0.47.7+ only. ### New features * Added [datetimeDiff](https://www.metabase.com/docs/latest/questions/query-builder/expressions/datetimediff) function support ([#117](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/117)) # 1.2.4 Metabase 0.47.7+ only. ### Bug fixes * Fixed UI question -> SQL conversion creating incorrect queries due to superfluous spaces in columns/tables/database names. # 1.2.3 ### Bug fixes * Fixed `LowCardinality(Nullable)` types introspection, where it was incorrectly reported as `type/*` ([#203](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/203)) # 1.2.2 ### Bug fixes * Removed forward slash from serialized IPv4/IPv6 columns. NB: IPv4/IPv6 columns are temporarily resolved as `type/TextLike` instead of `type/IPAddress` base type due to an unexpected result in Metabase 0.47 type check. * Removed superfluous CAST calls from generated queries that use Date* columns and/or intervals # 1.2.1 ### New features * Use HoneySQL2 in the driver # 1.2.0 ### New features * Metabase 0.47 support * Connection impersonation support (0.47 feature) ### Bug fixes * More correct general database type -> base type mapping * `DateTime64` is now correctly mapped to `:type/DateTime` * `database-required` field property is now correctly set to `true` if a field is not `Nullable` # 1.1.7 ### New features * JDBC driver upgrade (v0.4.1 -> [v0.4.6](https://github.com/ClickHouse/clickhouse-java/releases/tag/v0.4.6)) * Support DateTime64 by [@lucas-tubi](https://github.com/lucas-tubi) ([#165](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/165)) * Use native `startsWith`/`endsWith` instead of `LIKE str%`/`LIKE %str` # 1.1.6 ### Bug fixes * Fixed temporal bucketing issues (see [#155](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/155)) # 1.1.5 ### Bug fixes * Fixed Nippy error on cached questions (see [#147](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/147)) # 1.1.4 ### Bug fixes * Fixed `sum-where` behavior where previously it could not be applied to Int columns (see [#156](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/156)) # 1.1.3 ### New features * Hide `.inner` tables of Materialized Views. * Resolve `Map` base type to `type/Dictionary`. * Database name can now contain multiple schemas in the UI field (space-separated by default), which tells the driver to scan selected databases. Separator can be set in `metabase.driver.clickhouse/SEPARATOR`. (@veschin) # 1.1.2 ### Bug fixes * Now the driver is able to scan and work with `SimpleAggregateFunction` columns: those were excluded by mistake in 1.0.2. # 1.1.1 ### New features * Metabase 0.46.x compatibility. * Added [cljc.java-time](https://clojars.org/com.widdindustries/cljc.java-time) to dependencies, as it is no longer loaded by Metabase. # 1.1.0 ### New features * Update JDBC driver to v0.4.1. * Use new `product_name` additional option instead of `client_name` * Replace `sql-jdbc.execute/read-column [:clickhouse Types/ARRAY]` with `sql-jdbc.execute/read-column-thunk [:clickhouse Types/ARRAY]` to be compatible with Metabase 0.46 breaking changes once it is released. ### Bug fixes * Fix `sql-jdbc.execute/read-column-thunk [:clickhouse Types/TIME]` return type. # 1.0.4 ### New features * Adds a new "Scan all databases" UI toggle (disabled by default), which tells the driver to scan all available databases (excluding `system` and `information_schema`) instead of only the database it is connected to. * Database input moved below host/port/username/password in the UI. # 1.0.3 ### Bug fixes * Fixed NPE that could be thrown by the driver in case of empty database name input. # 1.0.2 ### Bug fixes * As the underlying JDBC driver version does not support columns with `(Simple)AggregationFunction` type, these columns are now excluded from the table metadata and data browser result sets to prevent sync or data browsing errors. # 1.0.1 ### Bug fixes * Boolean base type inference fix by [@s-huk](https://github.com/s-huk) (see [#134](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/134)) # 1.0.0 Formal stable release milestone. ### New features * Added HTTP User-Agent (via clickhouse-jdbc `client_name` setting) with the plugin info according to the [language client spec](https://docs.google.com/document/d/1924Dvy79KXIhfqKpi1EBVY3133pIdoMwgCQtZ-uhEKs/edit#heading=h.ah33hoz5xei2) # 0.9.2 ### New features * Allow to bypass system-wide proxy settings [#120](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/120) It's the first plugin release from the ClickHouse organization. From now on, the plugin is distributed under the Apache 2.0 License. # 0.9.1 ### New features * Metabase 0.45.x compatibility [#107](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/107) * Added SSH tunnel option [#116](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/116) # 0.9.0 ### New features * Using https://github.com/ClickHouse/clickhouse-jdbc `v0.3.2-patch11` ### Bug fixes * URLs with underscores [#23](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/23) * `now()` timezones issues [#81](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/81) * Boolean errors [#88](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/88) NB: there are messages like this in the Metabase logs ``` 2022-12-07 11:20:58,056 WARN internal.ClickHouseConnectionImpl :: [JDBC Compliant Mode] Transaction is not supported. You may change jdbcCompliant to false to throw SQLException instead. 2022-12-07 11:20:58,056 WARN internal.ClickHouseConnectionImpl :: [JDBC Compliant Mode] Transaction [ce0e121a-419a-4414-ac39-30f79eff7afd] (0 queries & 0 savepoints) is committed. ``` Unfortunately, this is the behaviour of the underlying JDBC driver now. Please consider raising the log level for `com.clickhouse.jdbc.internal.ClickHouseConnectionImpl` to `ERROR`. # 0.8.3 ### New features * Enable additional options for ClickHouse connection # 0.8.2 ### New features * Compatibility with Metabase 0.44 ================================================ FILE: CONTRIBUTING.md ================================================ ## Getting started ClickHouse driver for Metabase is an open-source project, and we welcome any contributions from the community. Please share your ideas, contribute to the codebase, and help us maintain up-to-date documentation. * Please report any issues you encounter during operations. * Feel free to create a pull request, preferably with a test or five. ## Setting up a development environment ### Requirements * Clojure 1.11+ * OpenJDK 17 * Node.js 16.x * Yarn For testing: Docker Compose Please refer to the extensive documentation available on the Metabase website: [Guide to writing a Metabase driver](https://www.metabase.com/docs/latest/developers-guide/drivers/start.html) ClickHouse driver's code should be inside the main Metabase repository checkout in `modules/drivers/clickhouse` directory. Additionally, you need to tweak Metabase's `deps.edn` a bit. The easiest way to set up a development environment is as follows (mostly the same as in the [CI](https://github.com/enqueue/metabase-clickhouse-driver/blob/master/.github/workflows/check.yml)): * Clone Metabase and ClickHouse driver repositories ```bash git clone https://github.com/metabase/metabase.git cd metabase git clone https://github.com/enqueue/metabase-clickhouse-driver.git modules/drivers/clickhouse ``` * Create custom Clojure profiles ```bash mkdir -p ~/.clojure cat modules/drivers/clickhouse/.github/deps.edn | sed -e "s|PWD|$PWD|g" > ~/.clojure/deps.edn ``` Modifying `~/.clojure/deps.edn` will create a new profile: `user/clickhouse`, that adds driver's sources to the class path, and includes all the Metabase tests that are guaranteed to work with the driver. * Install the Metabase dependencies: ```bash clojure -X:deps:drivers prep ``` * Build the frontend: ```bash yarn && yarn build-static-viz ``` * Add /etc/hosts entry Required for TLS tests. ```bash sudo -- sh -c "echo 127.0.0.1 server.clickhouseconnect.test >> /etc/hosts" ``` * Start Docker containers ```bash docker compose -f modules/drivers/clickhouse/docker-compose.yml up -d ``` Here's an overview of the started containers, which have the ports exposed to the `localhost` (see [docker-compose.yml](./docker-compose.yml)): - Metabase with the ClickHouse driver loaded from the JAR file (port: 3000) - Current ClickHouse version (port: 8123) - the main instance for all tests. - Current ClickHouse cluster with two nodes (+ nginx as an LB, port: 8127) - required for the set role tests (verifying that the role is set correctly via the query parameters). - Current ClickHouse version with TLS support (port: 8443) - required for the TLS tests. - Older ClickHouse version (port: 8124) - required for the string functions tests (switch between UTF8 (current) and non-UTF8 (pre-23.8) versions), as well as to verify that certain features, such as connection impersonation, are disabled on the older server versions. Now, you should be able to run the tests: ```bash DRIVERS=clickhouse clojure -X:dev:drivers:drivers-dev:test:user/clickhouse ``` you can see that we have our `:user/clickhouse` profile added to the command above, and with `DRIVERS=clickhouse` we instruct Metabase to run the tests only for ClickHouse. NB: Omitting `DRIVERS` will run the tests for all the built-in database drivers. If you want to run tests for only a specific namespace: ```bash DRIVERS=clickhouse clojure -X:dev:drivers:drivers-dev:test:user/clickhouse :only metabase.driver.clickhouse-test ``` or even a single test: ```bash DRIVERS=clickhouse clojure -X:dev:drivers:drivers-dev:test:user/clickhouse :only metabase.driver.clickhouse-test/clickhouse-nullable-arrays ``` Testing the driver with the older ClickHouse version (see [docker-compose.yml](./docker-compose.yml)): ```bash MB_CLICKHOUSE_TEST_PORT=8124 DRIVERS=clickhouse clojure -X:dev:drivers:drivers-dev:test:user/clickhouse :only metabase.driver.clickhouse-test ``` ## Building a jar You need to add an entry for ClickHouse in `modules/drivers/deps.edn` ```clj {:deps {... metabase/clickhouse {:local/root "clickhouse"} ...}} ``` or just run this from the root Metabase directory, overwriting the entire file: ```bash echo "{:deps {metabase/clickhouse {:local/root \"clickhouse\" }}}" > modules/drivers/deps.edn ``` Now, you should be able to build the final jar: ```bash bin/build-driver.sh clickhouse ``` As the result, `resources/modules/clickhouse.metabase-driver.jar` should be created. For smoke testing, there is a Metabase with the link to the driver available as a Docker container: ```bash docker compose -f modules/drivers/clickhouse/docker-compose.yml up -d metabase ``` It should pick up the driver jar as a volume. ================================================ FILE: HISTORY.md ================================================ # History The request for a ClickHouse Metabase driver is formulated in [Metabase issue #3332](https://github.com/metabase/metabase/issues/3332). Some impatient ClickHouse users started development. The Metabase team is asking driver developers to publish plug-ins and collect some experiences before considering a PR, so here we are. This driver is based on the following PRs: * [metabase#8491](https://github.com/metabase/metabase/pull/8491) * [metabase#8722](https://github.com/metabase/metabase/pull/8722) * [metabase#9469](https://github.com/metabase/metabase/pull/9469) ================================================ FILE: LICENSE ================================================ Copyright 2016-2025 ClickHouse, Inc. Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2016-2025 ClickHouse, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: README.md ================================================

ClickHouse driver for Metabase


## About The Metabase team promoted the ClickHouse driver to the core level as of Metabase 54 ([release notes](https://github.com/metabase/metabase/releases/tag/v0.54.1), [driver source](https://github.com/metabase/metabase/tree/v0.54.x/modules/drivers/clickhouse)). For the end user, this means the following: - Installing the driver manually is unnecessary, as it is now bundled with Metabase. - Starting from April 2025, the Metabase team will continue maintaining the driver. Please report new issues in [the main Metabase repository](https://github.com/metabase/metabase/issues). The ClickHouse team recommends avoiding older Metabase versions (53.x and earlier) with manual driver installation; instead, please use the updated Metabase distribution with the driver built-in. ## History The request for a ClickHouse Metabase driver was formulated in 2016 in [Metabase issue #3332](https://github.com/metabase/metabase/issues/3332). Several impatient ClickHouse users started the development in the main Metabase repo. In March 2019, after releasing the plugin SDK, the Metabase team [asked to publish the driver separately](https://github.com/metabase/metabase/pull/8491#issuecomment-471721980) in its own repository, and later that month, with Felix Mueller ([@enqueue](https://github.com/enqueue)) leading the efforts, the [initial version of the driver](https://github.com/ClickHouse/metabase-clickhouse-driver/releases/tag/v0.1) was out. The original implementation of the driver was based on the following pull requests: - [metabase#8491](https://github.com/metabase/metabase/pull/8491) - [metabase#8722](https://github.com/metabase/metabase/pull/8722) - [metabase#9469](https://github.com/metabase/metabase/pull/9469) The source base in these PRs comprises major contributions from these authors: - [@tsl-karlp](https://github.com/tsl-karlp) - Andrew Grigorev ([@ei-grad](https://github.com/ei-grad)) - Bogdan Mukvich ([@Badya](https://github.com/Badya)) - Felix Mueller ([@enqueue](https://github.com/enqueue)) > [!NOTE] > Special thanks to Felix Mueller ([@enqueue](https://github.com/enqueue)), who was the sole maintainer of the project from 2019 to 2022 before transferring it to ClickHouse. Starting from November 2022, Serge Klochkov ([@slvrtrn](https://github.com/slvrtrn)) joined as a maintainer. In early 2023, the repository was transferred to the ClickHouse organization, promoting it as an [official integration](https://clickhouse.com/blog/metabase-clickhouse-plugin-ga-release). Around that time, the driver also became available in [Metabase Cloud](https://www.metabase.com/cloud). In April 2025, the driver source code [was moved](https://github.com/metabase/metabase/pull/54740) to the main Metabase repository. Since [Metabase 54](https://github.com/metabase/metabase/releases/tag/v0.54.1), it is now available as a part of the official Metabase bundle. ================================================ FILE: build_docker_image.sh ================================================ #!/bin/bash if [ $# -lt 3 ]; then echo echo "Usage: ./build_docker_image.sh METABASE_VERSION CLICKHOUSE_DRIVER_VERSION DOCKER_IMAGE_TAG" echo echo "This script builds and tags a Metabase Docker image with ClickHouse driver built-in" echo echo "Example:" echo echo "./build_docker_image.sh v0.44.6 0.8.3 my-metabase-with-clickhouse:v0.0.1" exit 1 fi export DOWNLOAD_URL="https://github.com/ClickHouse/metabase-clickhouse-driver/releases/download/$2/clickhouse.metabase-driver.jar" echo "Downloading the driver from $DOWNLOAD_URL" cd .build curl -L -o clickhouse.metabase-driver.jar $DOWNLOAD_URL docker build --build-arg METABASE_VERSION=$1 --tag $3 . rm clickhouse.metabase-driver.jar ================================================ FILE: deps.edn ================================================ {:paths ["src" "resources"] :deps {com.widdindustries/cljc.java-time {:mvn/version "0.1.21"} com.clickhouse/clickhouse-jdbc {:mvn/version "0.8.4"} org.lz4/lz4-java {:mvn/version "1.8.0"}}} ================================================ FILE: docker-compose.yml ================================================ services: ########################################################################################################## # ClickHouse single node (CH driver + Metabase tests) ########################################################################################################## clickhouse: image: 'clickhouse/clickhouse-server:25.2-alpine' container_name: 'metabase-driver-clickhouse-server' hostname: clickhouse ports: - '8123:8123' - '9000:9000' environment: CLICKHOUSE_SKIP_USER_SETUP: 1 ulimits: nofile: soft: 262144 hard: 262144 volumes: - './.docker/clickhouse/single_node/config.xml:/etc/clickhouse-server/config.xml' - './.docker/clickhouse/single_node/users.xml:/etc/clickhouse-server/users.xml' ########################################################################################################## # ClickHouse single node (CH driver TLS tests only) ########################################################################################################## clickhouse_tls: build: context: ./ dockerfile: .docker/clickhouse/single_node_tls/Dockerfile container_name: 'metabase-driver-clickhouse-server-tls' ports: - '8443:8443' - '9440:9440' environment: CLICKHOUSE_SKIP_USER_SETUP: 1 ulimits: nofile: soft: 262144 hard: 262144 volumes: - './.docker/clickhouse/single_node_tls/config.xml:/etc/clickhouse-server/config.xml' - './.docker/clickhouse/single_node_tls/users.xml:/etc/clickhouse-server/users.xml' hostname: server.clickhouseconnect.test ########################################################################################################## # Older ClickHouse version (CH driver tests only) # For testing pre-23.8 string functions switch between UTF8 and non-UTF8 versions (see clickhouse_qp.clj) ########################################################################################################## clickhouse_older_version: image: 'clickhouse/clickhouse-server:23.3-alpine' container_name: 'metabase-driver-clickhouse-server-older-version' hostname: clickhouse.older ports: - '8124:8123' - '9001:9000' environment: CLICKHOUSE_SKIP_USER_SETUP: 1 ulimits: nofile: soft: 262144 hard: 262144 volumes: - './.docker/clickhouse/single_node/config.xml:/etc/clickhouse-server/config.xml' - './.docker/clickhouse/single_node/users.xml:/etc/clickhouse-server/users.xml' ########################################################################################################## # ClickHouse cluster (CH driver SET ROLE tests only) # See test/metabase/driver/clickhouse_set_role.clj ########################################################################################################## clickhouse_cluster_node1: image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-25.2-alpine}' ulimits: nofile: soft: 262144 hard: 262144 hostname: clickhouse1 container_name: metabase-driver-clickhouse-cluster-node-1 ports: - '8125:8123' - '9002:9000' - '9181:9181' environment: CLICKHOUSE_SKIP_USER_SETUP: 1 volumes: - './.docker/clickhouse/cluster/server1_config.xml:/etc/clickhouse-server/config.xml' - './.docker/clickhouse/cluster/server1_macros.xml:/etc/clickhouse-server/config.d/macros.xml' - './.docker/clickhouse/users.xml:/etc/clickhouse-server/users.xml' clickhouse_cluster_node2: image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-25.2-alpine}' ulimits: nofile: soft: 262144 hard: 262144 hostname: clickhouse2 container_name: metabase-driver-clickhouse-cluster-node-2 ports: - '8126:8123' - '9003:9000' - '9182:9181' environment: CLICKHOUSE_SKIP_USER_SETUP: 1 volumes: - './.docker/clickhouse/cluster/server2_config.xml:/etc/clickhouse-server/config.xml' - './.docker/clickhouse/cluster/server2_macros.xml:/etc/clickhouse-server/config.d/macros.xml' - './.docker/clickhouse/users.xml:/etc/clickhouse-server/users.xml' # Using Nginx as a cluster entrypoint and a round-robin load balancer for HTTP requests # See .docker/nginx/local.conf for the configuration nginx: image: 'nginx:1.23.1-alpine' hostname: nginx ports: - '8127:8123' volumes: - './.docker/nginx/local.conf:/etc/nginx/conf.d/local.conf' container_name: metabase-nginx ########################################################################################################## # Metabase ########################################################################################################## metabase: image: metabase/metabase-enterprise:v1.53.6.4 container_name: metabase-with-clickhouse-driver hostname: metabase environment: 'MB_HTTP_TIMEOUT': '5000' 'JAVA_TIMEZONE': 'UTC' ports: - '3000:3000' volumes: - '../../../resources/modules/clickhouse.metabase-driver.jar:/plugins/clickhouse.jar' - './.docker/clickhouse/single_node_tls/certificates/ca.crt:/certs/ca.crt' healthcheck: test: curl --fail -X GET -I http://localhost:3000/api/health || exit 1 interval: 15s timeout: 5s retries: 10 setup: build: .docker/setup/. container_name: metabase-clickhouse-setup volumes: - .docker/setup/setup.py:/app/setup.py depends_on: metabase: condition: service_healthy command: python /app/setup.py environment: host: http://metabase port: 3000 admin_email: 'admin@example.com' user_email: 'user@example.com' password: 'metabot1' ================================================ FILE: resources/metabase-plugin.yaml ================================================ info: name: Metabase ClickHouse Driver version: 1.53.4 description: Allows Metabase to connect to ClickHouse databases. contact-info: name: ClickHouse address: https://github.com/ClickHouse/metabase-clickhouse-driver driver: name: clickhouse display-name: ClickHouse lazy-load: true parent: sql-jdbc connection-properties: - host - merge: - port - default: 8123 - user - password - name: dbname display-name: Databases placeholder: default helper-text: "To specify multiple databases, separate them by the space symbol. For example: default data logs." - name: scan-all-databases display-name: Scan all databases type: boolean default: false description: Scan all tables from all available ClickHouse databases except the system ones. - ssl - ssh-tunnel - advanced-options-start - name: use-no-proxy display-name: Disable system wide proxy settings default: false type: boolean visible-if: advanced-options: true - name: clickhouse-settings display-name: ClickHouse settings (comma-separated) placeholder: "allow_experimental_analyzer=1,max_result_rows=100" visible-if: advanced-options: true - name: max-open-connections display-name: "Max open HTTP connections in the JDBC driver (default: 100)" placeholder: 100 visible-if: advanced-options: true - merge: - additional-options - placeholder: "connection_timeout=1000&socket_timeout=300000" - default-advanced-options connection-properties-include-tunnel-config: true init: - step: load-namespace namespace: metabase.driver.clickhouse - step: register-jdbc-driver class: com.clickhouse.jdbc.ClickHouseDriver ================================================ FILE: src/metabase/driver/clickhouse.clj ================================================ (ns metabase.driver.clickhouse "Driver for ClickHouse databases" #_{:clj-kondo/ignore [:unsorted-required-namespaces]} (:require [clojure.core.memoize :as memoize] [clojure.string :as str] [metabase.config :as config] [metabase.driver :as driver] [metabase.driver.clickhouse-introspection] [metabase.driver.clickhouse-nippy] [metabase.driver.clickhouse-qp] [metabase.driver.clickhouse-version :as clickhouse-version] [metabase.driver.ddl.interface :as ddl.i] [metabase.driver.sql :as driver.sql] [metabase.driver.sql-jdbc :as sql-jdbc] [metabase.driver.sql-jdbc.common :as sql-jdbc.common] [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn] [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute] [metabase.driver.sql.util :as sql.u] [metabase.lib.metadata :as lib.metadata] [metabase.query-processor.store :as qp.store] [metabase.upload :as upload] [metabase.util :as u] [metabase.util.log :as log]) (:import [com.clickhouse.client.api.query QuerySettings])) (set! *warn-on-reflection* true) (System/setProperty "clickhouse.jdbc.v2" "true") (driver/register! :clickhouse :parent #{:sql-jdbc}) (defmethod driver/display-name :clickhouse [_] "ClickHouse") (def ^:private product-name "metabase/1.53.4") (defmethod driver/prettify-native-form :clickhouse [_ native-form] (sql.u/format-sql-and-fix-params :mysql native-form)) (doseq [[feature supported?] {:standard-deviation-aggregations true :now true :set-timezone true :convert-timezone false :test/jvm-timezone-setting false :test/date-time-type false :test/time-type false :schemas true :datetime-diff true :upload-with-auto-pk false :window-functions/offset false :window-functions/cumulative (not config/is-test?) :left-join (not config/is-test?) :describe-fks false :actions false :metadata/key-constraints (not config/is-test?)}] (defmethod driver/database-supports? [:clickhouse feature] [_driver _feature _db] supported?)) (def ^:private default-connection-details {:user "default" :password "" :dbname "default" :host "localhost" :port "8123"}) (defn- connection-details->spec* [details] (let [;; ensure defaults merge on top of nils details (reduce-kv (fn [m k v] (assoc m k (or v (k default-connection-details)))) default-connection-details details) {:keys [user password dbname host port ssl clickhouse-settings max-open-connections]} details ;; if multiple databases were specified for the connection, ;; use only the first dbname as the "main" one dbname (first (str/split (str/trim dbname) #" ")) host (cond ; JDBCv1 used to accept schema in the `host` configuration option (str/starts-with? host "http://") (subs host 7) (str/starts-with? host "https://") (subs host 8) :else host)] (-> {:classname "com.clickhouse.jdbc.ClickHouseDriver" :subprotocol "clickhouse" :subname (str "//" host ":" port "/" dbname) :password (or password "") :user user :ssl (boolean ssl) :use_server_time_zone_for_dates true :product_name product-name :remember_last_set_roles true :http_connection_provider "HTTP_URL_CONNECTION" :jdbc_ignore_unsupported_values "true" :jdbc_schema_term "schema" :max_open_connections (or max-open-connections 100) ;; see also: https://clickhouse.com/docs/en/integrations/java#configuration :custom_http_params (or clickhouse-settings "")} (sql-jdbc.common/handle-additional-options details :separator-style :url)))) (defmethod sql-jdbc.execute/do-with-connection-with-options :clickhouse [driver db-or-id-or-spec {:keys [^String session-timezone _write?] :as options} f] (sql-jdbc.execute/do-with-resolved-connection driver db-or-id-or-spec options (fn [^java.sql.Connection conn] (when-not (sql-jdbc.execute/recursive-connection?) (when session-timezone (let [^com.clickhouse.jdbc.ConnectionImpl clickhouse-conn (.unwrap conn com.clickhouse.jdbc.ConnectionImpl) query-settings (new QuerySettings)] (.setOption query-settings "session_timezone" session-timezone) (.setDefaultQuerySettings clickhouse-conn query-settings))) (sql-jdbc.execute/set-best-transaction-level! driver conn) (sql-jdbc.execute/set-time-zone-if-supported! driver conn session-timezone) (when-let [db (cond ;; id? (integer? db-or-id-or-spec) (qp.store/with-metadata-provider db-or-id-or-spec (lib.metadata/database (qp.store/metadata-provider))) ;; db? (u/id db-or-id-or-spec) db-or-id-or-spec ;; otherwise it's a spec and we can't get the db :else nil)] (sql-jdbc.execute/set-role-if-supported! driver conn db))) (f conn)))) (def ^:private ^{:arglists '([db-details])} cloud? "Returns true if the `db-details` are for a ClickHouse Cloud instance, and false otherwise. If it fails to connect to the database, it throws a java.sql.SQLException." (memoize/ttl (fn [db-details] (let [spec (connection-details->spec* db-details)] (sql-jdbc.execute/do-with-connection-with-options :clickhouse spec nil (fn [^java.sql.Connection conn] (with-open [stmt (.createStatement conn) rset (.executeQuery stmt "SELECT value='1' FROM system.settings WHERE name='cloud_mode'")] (if (.next rset) (.getBoolean rset 1) false)))))) ;; cache the results for 48 hours; TTL is here only to eventually clear out old entries :ttl/threshold (* 48 60 60 1000))) (defmethod sql-jdbc.conn/connection-details->spec :clickhouse [_ details] (cond-> (connection-details->spec* details) (try (cloud? details) (catch java.sql.SQLException _e false)) ;; select_sequential_consistency guarantees that we can query data from any replica in CH Cloud ;; immediately after it is written (assoc :select_sequential_consistency true))) (defmethod driver/database-supports? [:clickhouse :uploads] [_driver _feature db] (if (:details db) (try (cloud? (:details db)) (catch java.sql.SQLException _e false)) false)) (defmethod driver/can-connect? :clickhouse [driver details] (if config/is-test? (try ;; Default SELECT 1 is not enough for Metabase test suite, ;; as it works slightly differently than expected there (let [spec (sql-jdbc.conn/connection-details->spec driver details) db (ddl.i/format-name driver (or (:dbname details) (:db details) "default"))] (sql-jdbc.execute/do-with-connection-with-options driver spec nil (fn [^java.sql.Connection conn] (let [stmt (.prepareStatement conn "SELECT count(*) > 0 FROM system.databases WHERE name = ?") _ (.setString stmt 1 db) rset (.executeQuery stmt)] (when (.next rset) (.getBoolean rset 1)))))) (catch Throwable e (log/error e "An exception during ClickHouse connectivity check") false)) ;; During normal usage, fall back to the default implementation (sql-jdbc.conn/can-connect? driver details))) (defmethod driver/db-default-timezone :clickhouse [driver database] (sql-jdbc.execute/do-with-connection-with-options driver database nil (fn [^java.sql.Connection conn] (with-open [stmt (.createStatement conn) rset (.executeQuery stmt "SELECT timezone() AS tz")] (when (.next rset) (.getString rset 1)))))) (defmethod driver/db-start-of-week :clickhouse [_] :monday) (defmethod ddl.i/format-name :clickhouse [_ table-or-field-name] (when table-or-field-name (str/replace table-or-field-name #"-" "_"))) ;;; ------------------------------------------ Connection Impersonation ------------------------------------------ (defmethod driver/upload-type->database-type :clickhouse [_driver upload-type] (case upload-type ::upload/varchar-255 "Nullable(String)" ::upload/text "Nullable(String)" ::upload/int "Nullable(Int64)" ::upload/float "Nullable(Float64)" ::upload/boolean "Nullable(Boolean)" ::upload/date "Nullable(Date32)" ::upload/datetime "Nullable(DateTime64(3))" ::upload/offset-datetime nil)) (defmethod driver/table-name-length-limit :clickhouse [_driver] ;; FIXME: This is a lie because you're really limited by a filesystems' limits, because Clickhouse uses ;; filenames as table/column names. But its an approximation 206) (defn- quote-name [s] (let [parts (str/split (name s) #"\.")] (str/join "." (map #(str "`" % "`") parts)))) (defn- create-table!-sql "Creates a ClickHouse table with the given name and column definitions. It assumes the engine is MergeTree, so it only works with Clickhouse Cloud and single node on-premise deployments at the moment." [_driver table-name column-definitions & {:keys [primary-key] :as opts}] (str/join "\n" [(#'sql-jdbc/create-table!-sql :sql-jdbc table-name column-definitions opts) "ENGINE = MergeTree" (format "ORDER BY (%s)" (str/join ", " (map quote-name primary-key))) ;; disable insert idempotency to allow duplicate inserts "SETTINGS replicated_deduplication_window = 0"])) (defmethod driver/create-table! :clickhouse [driver db-id table-name column-definitions & {:keys [primary-key]}] (sql-jdbc.execute/do-with-connection-with-options driver db-id {:write? true} (fn [^java.sql.Connection conn] (with-open [stmt (.createStatement conn)] (.execute stmt (create-table!-sql driver table-name column-definitions :primary-key primary-key)))))) (defmethod driver/insert-into! :clickhouse [driver db-id table-name column-names values] (when (seq values) (sql-jdbc.execute/do-with-connection-with-options driver db-id {:write? true} (fn [^java.sql.Connection conn] (let [sql (format "INSERT INTO %s (%s) VALUES (%s)" (quote-name table-name) (str/join ", " (map quote-name column-names)) (str/join ", " (repeat (count column-names) "?")))] (with-open [ps (.prepareStatement conn sql)] (doseq [row values] (when (seq row) (doseq [[idx v] (map-indexed (fn [x y] [(inc x) y]) row)] (condp isa? (type v) nil (.setString ps idx nil) java.lang.String (.setString ps idx v) java.lang.Boolean (.setBoolean ps idx v) java.lang.Long (.setLong ps idx v) java.lang.Double (.setFloat ps idx v) java.math.BigInteger (.setObject ps idx v) java.time.LocalDate (.setObject ps idx v) java.time.LocalDateTime (.setObject ps idx v) java.time.OffsetDateTime (.setObject ps idx v) (.setString ps idx (str v)))) (.addBatch ps))) (doall (.executeBatch ps)))))))) ;;; ------------------------------------------ User Impersonation ------------------------------------------ (defmethod driver/database-supports? [:clickhouse :connection-impersonation] [_driver _feature db] (if db (try (clickhouse-version/is-at-least? 24 4 db) (catch Throwable _e false)) false)) (defmethod driver.sql/set-role-statement :clickhouse [_ role] (let [default-role (driver.sql/default-database-role :clickhouse nil) quote-if-needed (fn [r] (if (or (re-matches #"\".*\"" r) (= role default-role)) r (format "\"%s\"" r))) quoted-role (->> (str/split role #",") (map quote-if-needed) (str/join ",")) statement (format "SET ROLE %s" quoted-role)] statement)) (defmethod driver.sql/default-database-role :clickhouse [_ _] "NONE") ================================================ FILE: src/metabase/driver/clickhouse_introspection.clj ================================================ (ns metabase.driver.clickhouse-introspection (:require [clojure.java.jdbc :as jdbc] [clojure.string :as str] [metabase.config :as config] [metabase.driver :as driver] [metabase.driver.ddl.interface :as ddl.i] [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn] [metabase.driver.sql-jdbc.sync :as sql-jdbc.sync] [metabase.driver.sql-jdbc.sync.describe-table :as sql-jdbc.describe-table] [metabase.util :as u]) (:import (java.sql DatabaseMetaData))) (set! *warn-on-reflection* true) (def ^:private database-type->base-type (sql-jdbc.sync/pattern-based-database-type->base-type [[#"array" :type/Array] [#"bool" :type/Boolean] [#"date" :type/Date] [#"date32" :type/Date] [#"decimal" :type/Decimal] [#"enum8" :type/Text] [#"enum16" :type/Text] [#"fixedstring" :type/TextLike] [#"float32" :type/Float] [#"float64" :type/Float] [#"int8" :type/Integer] [#"int16" :type/Integer] [#"int32" :type/Integer] [#"int64" :type/BigInteger] [#"ipv4" :type/IPAddress] [#"ipv6" :type/IPAddress] [#"map" :type/Dictionary] [#"string" :type/Text] [#"tuple" :type/*] [#"uint8" :type/Integer] [#"uint16" :type/Integer] [#"uint32" :type/Integer] [#"uint64" :type/BigInteger] [#"uuid" :type/UUID]])) (defn- normalize-db-type [db-type] (cond ;; LowCardinality (str/starts-with? db-type "lowcardinality") (normalize-db-type (subs db-type 15 (- (count db-type) 1))) ;; Nullable (str/starts-with? db-type "nullable") (normalize-db-type (subs db-type 9 (- (count db-type) 1))) ;; for test purposes only: GMT0 is a legacy timezone; ;; it maps to LocalDateTime instead of OffsetDateTime ;; (= db-type "datetime64(3, 'gmt0')") ;; :type/DateTime ;; DateTime64 (str/starts-with? db-type "datetime64") :type/DateTimeWithLocalTZ ;; DateTime (str/starts-with? db-type "datetime") :type/DateTimeWithLocalTZ ;; Enum* (str/starts-with? db-type "enum") :type/Text ;; Map (str/starts-with? db-type "map") :type/Dictionary ;; Tuple (str/starts-with? db-type "tuple") :type/* ;; SimpleAggregateFunction (str/starts-with? db-type "simpleaggregatefunction") (normalize-db-type (subs db-type (+ (str/index-of db-type ",") 2) (- (count db-type) 1))) ;; _ :else (or (database-type->base-type (keyword db-type)) :type/*))) ;; Enum8(UInt8) -> :type/Text, DateTime64(Europe/Amsterdam) -> :type/DateTime, ;; Nullable(DateTime) -> :type/DateTime, SimpleAggregateFunction(sum, Int64) -> :type/BigInteger, etc (defmethod sql-jdbc.sync/database-type->base-type :clickhouse [_ database-type] (let [db-type (if (keyword? database-type) (subs (str database-type) 1) database-type)] (normalize-db-type (u/lower-case-en db-type)))) (defmethod sql-jdbc.sync/excluded-schemas :clickhouse [_] #{"system" "information_schema" "INFORMATION_SCHEMA"}) (def ^:private allowed-table-types (into-array String ["TABLE" "VIEW" "FOREIGN TABLE" "REMOTE TABLE" "DICTIONARY" "MATERIALIZED VIEW" "MEMORY TABLE" "LOG TABLE"])) (defn- tables-set [tables] (set (for [table tables] (let [remarks (:remarks table)] {:name (:table_name table) :schema (:table_schem table) :description (when-not (str/blank? remarks) remarks)})))) (defn- get-tables-from-metadata [^DatabaseMetaData metadata schema-pattern] (.getTables metadata nil ; catalog - unused in the source code there schema-pattern "%" ; tablePattern "%" = match all tables allowed-table-types)) (defn- not-inner-mv-table? [table] (not (str/starts-with? (:table_name table) ".inner"))) (defn- ->spec [db] (if (u/id db) (sql-jdbc.conn/db->pooled-connection-spec db) db)) (defn- get-all-tables [db] (jdbc/with-db-metadata [metadata (->spec db)] (->> (get-tables-from-metadata metadata "%") (jdbc/metadata-result) (vec) (filter #(and (not (contains? (sql-jdbc.sync/excluded-schemas :clickhouse) (:table_schem %))) (not-inner-mv-table? %))) (tables-set)))) ;; Strangely enough, the tests only work with :db keyword, ;; but the actual sync from the UI uses :dbname (defn- get-db-name [db] (or (get-in db [:details :dbname]) (get-in db [:details :db]))) (defn- get-tables-in-dbs [db-or-dbs] (->> (for [db (as-> (or (get-db-name db-or-dbs) "default") dbs (str/split dbs #" ") (remove empty? dbs) (map (comp #(ddl.i/format-name :clickhouse %) str/trim) dbs))] (jdbc/with-db-metadata [metadata (->spec db-or-dbs)] (jdbc/metadata-result (get-tables-from-metadata metadata db)))) (apply concat) (filter not-inner-mv-table?) (tables-set))) (defmethod driver/describe-database :clickhouse [_ {{:keys [scan-all-databases]} :details :as db}] {:tables (if (boolean scan-all-databases) (get-all-tables db) (get-tables-in-dbs db))}) (defn- ^:private is-db-required? [field] (not (str/starts-with? (get field :database-type) "Nullable"))) (defmethod driver/describe-table :clickhouse [_ database table] (let [table-metadata (sql-jdbc.sync/describe-table :clickhouse database table) filtered-fields (for [field (:fields table-metadata) :let [updated-field (update field :database-required (fn [_] (is-db-required? field)))] ;; Skip all AggregateFunction (but keeping SimpleAggregateFunction) columns ;; JDBC does not support that and it crashes the data browser :when (not (re-matches #"^AggregateFunction\(.+$" (get field :database-type)))] updated-field)] (merge table-metadata {:fields (set filtered-fields)}))) (defmethod sql-jdbc.describe-table/get-table-pks :clickhouse [_driver ^java.sql.Connection conn db-name-or-nil table] ;; JDBC v2 sets the PKs now, so that :metadata/key-constraints feature should be enabled; ;; however, enabling :metadata/key-constraints will also enable left-join tests which are currently failing (if (not config/is-test?) (sql-jdbc.describe-table/get-table-pks :sql-jdbc conn db-name-or-nil table) [])) ================================================ FILE: src/metabase/driver/clickhouse_nippy.clj ================================================ (ns metabase.driver.clickhouse-nippy (:require [taoensso.nippy :as nippy]) (:import [java.io DataInput DataOutput])) (set! *warn-on-reflection* false) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; com.clickhouse.data.value.UnsignedByte ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (nippy/extend-freeze com.clickhouse.data.value.UnsignedByte :clickhouse/UnsignedByte [^com.clickhouse.data.value.UnsignedByte x ^DataOutput data-output] ;; can't enable *warn-on-reflection* because of this call (nippy/freeze-to-out! data-output (.toString x))) (nippy/extend-thaw :clickhouse/UnsignedByte [^DataInput data-input] (com.clickhouse.data.value.UnsignedByte/valueOf (nippy/thaw-from-in! data-input))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; com.clickhouse.data.value.UnsignedShort ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (nippy/extend-freeze com.clickhouse.data.value.UnsignedShort :clickhouse/UnsignedShort [^com.clickhouse.data.value.UnsignedShort x ^DataOutput data-output] (nippy/freeze-to-out! data-output (.toString x))) (nippy/extend-thaw :clickhouse/UnsignedShort [^DataInput data-input] (com.clickhouse.data.value.UnsignedShort/valueOf (nippy/thaw-from-in! data-input))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; com.clickhouse.data.value.UnsignedInteger ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (nippy/extend-freeze com.clickhouse.data.value.UnsignedInteger :clickhouse/UnsignedInteger [^com.clickhouse.data.value.UnsignedInteger x ^DataOutput data-output] (nippy/freeze-to-out! data-output (.toString x))) (nippy/extend-thaw :clickhouse/UnsignedInteger [^DataInput data-input] (com.clickhouse.data.value.UnsignedInteger/valueOf (nippy/thaw-from-in! data-input))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; com.clickhouse.data.value.UnsignedLong ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (nippy/extend-freeze com.clickhouse.data.value.UnsignedLong :clickhouse/UnsignedLong [^com.clickhouse.data.value.UnsignedLong x ^DataOutput data-output] (nippy/freeze-to-out! data-output (.toString x))) (nippy/extend-thaw :clickhouse/UnsignedLong [^DataInput data-input] (com.clickhouse.data.value.UnsignedLong/valueOf (nippy/thaw-from-in! data-input))) ================================================ FILE: src/metabase/driver/clickhouse_qp.clj ================================================ (ns metabase.driver.clickhouse-qp "CLickHouse driver: QueryProcessor-related definition" #_{:clj-kondo/ignore [:unsorted-required-namespaces]} (:require [clojure.string :as str] [honey.sql :as sql] [java-time.api :as t] [metabase.driver.clickhouse-nippy] [metabase.driver.clickhouse-version :as clickhouse-version] [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute] [metabase.driver.sql.query-processor :as sql.qp :refer [add-interval-honeysql-form]] [metabase.driver.sql.util :as sql.u] [metabase.driver.sql.util.unprepare :as unprepare] [metabase.legacy-mbql.util :as mbql.u] [metabase.query-processor.timezone :as qp.timezone] [metabase.util :as u] [metabase.util.date-2 :as u.date] [metabase.util.honey-sql-2 :as h2x]) (:import [java.sql ResultSet ResultSetMetaData Types] [java.time LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime] java.util.Arrays)) ;; (set! *warn-on-reflection* true) ;; isn't enabled because of Arrays/toString call (defmethod sql.qp/quote-style :clickhouse [_] :mysql) ;; without try, there might be test failures when QP is not yet initialized ;; e.g., when a test is preparing the dataset (defn- get-report-timezone-id-safely [] (try (qp.timezone/report-timezone-id-if-supported) (catch Throwable _e nil))) ;; datetime('europe/amsterdam') -> europe/amsterdam (defn- extract-datetime-timezone [db-type] (when (and db-type (string? db-type)) (cond ;; e.g. DateTime64(3, 'Europe/Amsterdam') (str/starts-with? db-type "datetime64") (if (> (count db-type) 17) (subs db-type 15 (- (count db-type) 2)) nil) ;; e.g. DateTime('Europe/Amsterdam') (str/starts-with? db-type "datetime") (if (> (count db-type) 12) (subs db-type 10 (- (count db-type) 2)) nil) ;; _ :else nil))) (defn- remove-low-cardinality-and-nullable [db-type] (when (and db-type (string? db-type)) (let [db-type-lowercase (u/lower-case-en db-type) without-low-car (if (str/starts-with? db-type-lowercase "lowcardinality(") (subs db-type-lowercase 15 (- (count db-type-lowercase) 1)) db-type-lowercase) without-nullable (if (str/starts-with? without-low-car "nullable(") (subs without-low-car 9 (- (count without-low-car) 1)) without-low-car)] without-nullable))) (defn- in-report-timezone [expr] (let [report-timezone (get-report-timezone-id-safely) lower (u/lower-case-en (h2x/database-type expr)) db-type (remove-low-cardinality-and-nullable lower)] (if (and report-timezone (string? db-type) (str/starts-with? db-type "datetime")) (let [timezone (extract-datetime-timezone db-type)] (if (not (= timezone (u/lower-case-en report-timezone))) [:'toTimeZone expr (h2x/literal report-timezone)] expr)) expr))) (defmethod sql.qp/date [:clickhouse :default] [_ _ expr] expr) ;;; ------------------------------------------------------------------------------------ ;;; Extract functions ;;; ------------------------------------------------------------------------------------ (defn- date-extract [ch-fn expr db-type] (-> [ch-fn (in-report-timezone expr)] (h2x/with-database-type-info db-type))) (defmethod sql.qp/date [:clickhouse :day-of-week] [_ _ expr] ;; a tick in the function name prevents HSQL2 to make the function call UPPERCASE ;; https://cljdoc.org/d/com.github.seancorfield/honeysql/2.4.1011/doc/getting-started/other-databases#clickhouse (sql.qp/adjust-day-of-week :clickhouse (date-extract :'toDayOfWeek expr "uint8"))) (defmethod sql.qp/date [:clickhouse :month-of-year] [_ _ expr] (date-extract :'toMonth expr "uint8")) (defmethod sql.qp/date [:clickhouse :minute-of-hour] [_ _ expr] (date-extract :'toMinute expr "uint8")) (defmethod sql.qp/date [:clickhouse :hour-of-day] [_ _ expr] (date-extract :'toHour expr "uint8")) (defmethod sql.qp/date [:clickhouse :day-of-month] [_ _ expr] (date-extract :'toDayOfMonth expr "uint8")) (defmethod sql.qp/date [:clickhouse :day-of-year] [_ _ expr] (date-extract :'toDayOfYear expr "uint16")) (defmethod sql.qp/date [:clickhouse :week-of-year-iso] [_ _ expr] (date-extract :'toISOWeek expr "uint8")) (defmethod sql.qp/date [:clickhouse :quarter-of-year] [_ _ expr] (date-extract :'toQuarter expr "uint8")) (defmethod sql.qp/date [:clickhouse :year-of-era] [_ _ expr] (date-extract :'toYear expr "uint16")) ;;; ------------------------------------------------------------------------------------ ;;; Truncate functions ;;; ------------------------------------------------------------------------------------ (defn- date-trunc [ch-fn expr] [ch-fn (in-report-timezone expr)]) (defn- to-start-of-week [expr] (date-trunc :'toMonday expr)) (defmethod sql.qp/date [:clickhouse :minute] [_ _ expr] (date-trunc :'toStartOfMinute expr)) (defmethod sql.qp/date [:clickhouse :hour] [_ _ expr] (date-trunc :'toStartOfHour expr)) (defmethod sql.qp/date [:clickhouse :day] [_ _ expr] (date-trunc :'toStartOfDay expr)) (defmethod sql.qp/date [:clickhouse :week] [driver _ expr] (sql.qp/adjust-start-of-week driver to-start-of-week expr)) (defmethod sql.qp/date [:clickhouse :month] [_ _ expr] (date-trunc :'toStartOfMonth expr)) (defmethod sql.qp/date [:clickhouse :quarter] [_ _ expr] (date-trunc :'toStartOfQuarter expr)) (defmethod sql.qp/date [:clickhouse :year] [_ _ expr] (date-trunc :'toStartOfYear expr)) ;;; ------------------------------------------------------------------------------------ ;;; Unix timestamps functions ;;; ------------------------------------------------------------------------------------ (defmethod sql.qp/unix-timestamp->honeysql [:clickhouse :seconds] [_ _ expr] (h2x/->datetime expr)) (defmethod sql.qp/unix-timestamp->honeysql [:clickhouse :milliseconds] [_ _ expr] (let [report-timezone (get-report-timezone-id-safely) inner-expr (h2x// expr 1000)] (if report-timezone [:'toDateTime64 inner-expr 3 report-timezone] [:'toDateTime64 inner-expr 3]))) (defmethod sql.qp/unix-timestamp->honeysql [:clickhouse :microseconds] [_ _ expr] (let [report-timezone (get-report-timezone-id-safely) inner-expr [:'toInt64 (h2x// expr 1000)]] (if report-timezone [:'fromUnixTimestamp64Milli inner-expr report-timezone] [:'fromUnixTimestamp64Milli inner-expr]))) ;;; ------------------------------------------------------------------------------------ ;;; HoneySQL forms ;;; ------------------------------------------------------------------------------------ (defmethod sql.qp/->honeysql [:clickhouse :convert-timezone] [driver [_ arg target-timezone source-timezone]] (let [expr (sql.qp/->honeysql driver (cond-> arg (string? arg) u.date/parse)) with-tz-info? (h2x/is-of-type? expr #"(?:nullable\(|lowcardinality\()?(datetime64\(\d, {0,1}'.*|datetime\(.*)") _ (sql.u/validate-convert-timezone-args with-tz-info? target-timezone source-timezone)] (if (not with-tz-info?) [:'plus expr [:'toIntervalSecond [:'minus [:'timeZoneOffset [:'toTimeZone expr target-timezone]] [:'timeZoneOffset [:'toTimeZone expr source-timezone]]]]] [:'toTimeZone expr target-timezone]))) (defmethod sql.qp/current-datetime-honeysql-form :clickhouse [_] (let [report-timezone (get-report-timezone-id-safely) [expr db-type] (if report-timezone [[:'now64 [:raw 9] (h2x/literal report-timezone)] (format "DateTime64(9, '%s')" report-timezone)] [[:'now64 [:raw 9]] "DateTime64(9)"])] (h2x/with-database-type-info expr db-type))) (defn- date-time-parse-fn [nano] (if (zero? nano) :'parseDateTimeBestEffort :'parseDateTime64BestEffort)) (defmethod sql.qp/->honeysql [:clickhouse LocalDateTime] [_ ^java.time.LocalDateTime t] (let [formatted (t/format "yyyy-MM-dd HH:mm:ss.SSS" t) report-tz (or (get-report-timezone-id-safely) "UTC")] (if (zero? (.getNano t)) [:'parseDateTimeBestEffort formatted report-tz] [:'parseDateTime64BestEffort formatted 3 report-tz]))) (defmethod sql.qp/->honeysql [:clickhouse ZonedDateTime] [_ ^java.time.ZonedDateTime t] (let [formatted (t/format "yyyy-MM-dd HH:mm:ss.SSSZZZZZ" t) fn (date-time-parse-fn (.getNano t))] [fn formatted])) (defmethod sql.qp/->honeysql [:clickhouse OffsetDateTime] [_ ^java.time.OffsetDateTime t] ;; copy-paste due to reflection warnings (let [formatted (t/format "yyyy-MM-dd HH:mm:ss.SSSZZZZZ" t) fn (date-time-parse-fn (.getNano t))] [fn formatted])) (defmethod sql.qp/->honeysql [:clickhouse LocalDate] [_ ^java.time.LocalDate t] [:'parseDateTimeBestEffort t]) (defn- local-date-time [^java.time.LocalTime t] (t/local-date-time (t/local-date 1970 1 1) t)) (defmethod sql.qp/->honeysql [:clickhouse LocalTime] [driver ^java.time.LocalTime t] (sql.qp/->honeysql driver (local-date-time t))) (defmethod sql.qp/->honeysql [:clickhouse OffsetTime] [driver ^java.time.OffsetTime t] (sql.qp/->honeysql driver (t/offset-date-time (local-date-time (.toLocalTime t)) (.getOffset t)))) (defn- args->float64 [args] (map (fn [arg] [:'toFloat64 (sql.qp/->honeysql :clickhouse arg)]) args)) (defn- interval? [expr] (mbql.u/is-clause? :interval expr)) (defmethod sql.qp/->honeysql [:clickhouse :+] [driver [_ & args]] (if (some interval? args) (if-let [[field intervals] (u/pick-first (complement interval?) args)] (reduce (fn [hsql-form [_ amount unit]] (add-interval-honeysql-form driver hsql-form amount unit)) (sql.qp/->honeysql driver field) intervals) (throw (ex-info "Summing intervals is not supported" {:args args}))) (into [:+] (args->float64 args)))) (defmethod sql.qp/->honeysql [:clickhouse :log] [driver [_ field]] [:'log10 (sql.qp/->honeysql driver field)]) (defn- format-expr [expr] (first (sql/format-expr (sql.qp/->honeysql :clickhouse expr) {:nested true}))) (defmethod sql.qp/->honeysql [:clickhouse :percentile] [_ [_ field p]] [:raw (format "quantile(%s)(%s)" (format-expr p) (format-expr field))]) (defmethod sql.qp/->honeysql [:clickhouse :regex-match-first] [driver [_ arg pattern]] [:'extract (sql.qp/->honeysql driver arg) pattern]) (defmethod sql.qp/->honeysql [:clickhouse :stddev] [driver [_ field]] [:'stddevPop (sql.qp/->honeysql driver field)]) (defmethod sql.qp/->honeysql [:clickhouse :median] [driver [_ field]] [:'median (sql.qp/->honeysql driver field)]) ;; Substring does not work for Enums, so we need to cast to String (defmethod sql.qp/->honeysql [:clickhouse :substring] [driver [_ arg start length]] (let [str [:'toString (sql.qp/->honeysql driver arg)]] (if length [:'substring str (sql.qp/->honeysql driver start) (sql.qp/->honeysql driver length)] [:'substring str (sql.qp/->honeysql driver start)]))) (defmethod sql.qp/->honeysql [:clickhouse :var] [driver [_ field]] [:'varPop (sql.qp/->honeysql driver field)]) (defmethod sql.qp/->float :clickhouse [_ value] [:'toFloat64 value]) (defmethod sql.qp/->honeysql [:clickhouse :value] [driver value] (let [[_ value {base-type :base_type}] value] (when (some? value) (condp #(isa? %2 %1) base-type :type/IPAddress [:'toIPv4 value] (sql.qp/->honeysql driver value))))) (defmethod sql.qp/->honeysql [:clickhouse :=] [driver [op field value]] (let [[qual valuevalue fieldinfo] value hsql-field (sql.qp/->honeysql driver field) hsql-value (sql.qp/->honeysql driver value)] (if (and (isa? qual :value) (isa? (:base_type fieldinfo) :type/Text) (nil? valuevalue)) [:or [:= hsql-field hsql-value] [:= [:'empty hsql-field] 1]] ((get-method sql.qp/->honeysql [:sql :=]) driver [op field value])))) (defmethod sql.qp/->honeysql [:clickhouse :!=] [driver [op field value]] (let [[qual valuevalue fieldinfo] value hsql-field (sql.qp/->honeysql driver field) hsql-value (sql.qp/->honeysql driver value)] (if (and (isa? qual :value) (isa? (:base_type fieldinfo) :type/Text) (nil? valuevalue)) [:and [:!= hsql-field hsql-value] [:= [:'notEmpty hsql-field] 1]] ((get-method sql.qp/->honeysql [:sql :!=]) driver [op field value])))) ;; I do not know why the tests expect nil counts for empty results ;; but that's how it is :-) ;; ;; It would even be better if we could use countIf and sumIf directly ;; ;; metabase.query-processor-test.count-where-test ;; metabase.query-processor-test.share-test (defmethod sql.qp/->honeysql [:clickhouse :count-where] [driver [_ pred]] [:case [:> [:'count] 0] [:sum [:case (sql.qp/->honeysql driver pred) 1 :else 0]] :else nil]) (defmethod sql.qp/->honeysql [:clickhouse :sum-where] [driver [_ field pred]] [:sum [:case (sql.qp/->honeysql driver pred) (sql.qp/->honeysql driver field) :else 0]]) (defmethod sql.qp/add-interval-honeysql-form :clickhouse [_ dt amount unit] (h2x/+ dt [:raw (format "INTERVAL %d %s" (int amount) (name unit))])) (defn- clickhouse-string-fn [fn-name field value options] (let [hsql-field (sql.qp/->honeysql :clickhouse field) hsql-value (sql.qp/->honeysql :clickhouse value)] (if (get options :case-sensitive true) [fn-name hsql-field hsql-value] [fn-name [:'lowerUTF8 hsql-field] [:'lowerUTF8 hsql-value]]))) (defmethod sql.qp/->honeysql [:clickhouse :starts-with] [_ [_ field value options]] (let [starts-with (clickhouse-version/with-min 23 8 (constantly :'startsWithUTF8) (constantly :'startsWith))] (clickhouse-string-fn starts-with field value options))) (defmethod sql.qp/->honeysql [:clickhouse :ends-with] [_ [_ field value options]] (let [ends-with (clickhouse-version/with-min 23 8 (constantly :'endsWithUTF8) (constantly :'endsWith))] (clickhouse-string-fn ends-with field value options))) (defmethod sql.qp/->honeysql [:clickhouse :contains] [_ [_ field value options]] (let [hsql-field (sql.qp/->honeysql :clickhouse field) hsql-value (sql.qp/->honeysql :clickhouse value) position-fn (if (get options :case-sensitive true) :'positionUTF8 :'positionCaseInsensitiveUTF8)] [:> [position-fn hsql-field hsql-value] 0])) (defmethod sql.qp/->honeysql [:clickhouse :datetime-diff] [driver [_ x y unit]] (let [x (sql.qp/->honeysql driver x) y (sql.qp/->honeysql driver y)] (case unit ;; Week: Metabase tests expect a bit different result from what `age` provides (:week) [:'intDiv [:'dateDiff (h2x/literal :day) (date-trunc :'toStartOfDay x) (date-trunc :'toStartOfDay y)] [:raw 7]] ;; ------------------------- (:year :month :quarter :day) [:'age (h2x/literal unit) (date-trunc :'toStartOfDay x) (date-trunc :'toStartOfDay y)] ;; ------------------------- (:hour :minute :second) [:'age (h2x/literal unit) (in-report-timezone x) (in-report-timezone y)]))) ;; We do not have Time data types, so we cheat a little bit (defmethod sql.qp/cast-temporal-string [:clickhouse :Coercion/ISO8601->Time] [_driver _special_type expr] [:'parseDateTimeBestEffort [:'concat "1970-01-01T" expr]]) (defmethod sql.qp/cast-temporal-byte [:clickhouse :Coercion/ISO8601->Time] [_driver _special_type expr] expr) ;;; ------------------------------------------------------------------------------------ ;;; JDBC-related functions ;;; ------------------------------------------------------------------------------------ (defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/TINYINT] [_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i] (fn [] (.getObject rs i))) (defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/SMALLINT] [_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i] (fn [] (.getObject rs i))) ;; This is for tests only - some of them expect nil values ;; getInt/getLong return 0 in case of a NULL value in the result set ;; the only way to check if it was actually NULL - call ResultSet.wasNull afterwards (defn- with-null-check [^ResultSet rs value] (if (.wasNull rs) nil value)) (defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/BIGINT] [_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i] (fn [] (with-null-check rs (.getBigDecimal rs i)))) (defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/INTEGER] [_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i] (fn [] (with-null-check rs (.getLong rs i)))) (def ^:private utc-zone-id (java.time.ZoneId/of "UTC")) (defn- zdt-in-report-timezone [^ZonedDateTime zdt] (let [maybe-report-timezone (get-report-timezone-id-safely)] (if maybe-report-timezone (.withZoneSameInstant zdt (java.time.ZoneId/of maybe-report-timezone)) (if (= (.getId (.getZone zdt)) "GMT0") ;; for test purposes only; GMT0 is a legacy tz (.withZoneSameInstant zdt utc-zone-id) zdt)))) (defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/DATE] [_ ^ResultSet rs ^ResultSetMetaData _rsmeta ^Integer i] (fn [] (when-let [sql-date (.getDate rs i)] (.toLocalDate sql-date)))) (defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/TIMESTAMP] [_ ^ResultSet rs ^ResultSetMetaData rsmeta ^Integer i] (fn [] (when-let [zdt (.getObject rs i ZonedDateTime)] (let [db-type (remove-low-cardinality-and-nullable (.getColumnTypeName rsmeta i))] (if (= db-type "datetime64(3, 'gmt0')") ;; a hack for some MB test assertions only; GMT0 is a legacy tz (.toLocalDateTime (zdt-in-report-timezone zdt)) ;; this is the normal behavior (.toOffsetDateTime (.withZoneSameInstant (zdt-in-report-timezone zdt) utc-zone-id))))))) (defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/TIME] [_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i] (fn [] (.getObject rs i OffsetTime))) (defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/NUMERIC] [_ ^ResultSet rs ^ResultSetMetaData rsmeta ^Integer i] (fn [] ; count is NUMERIC cause UInt64 is too large for the canonical SQL BIGINT, ; and defaults to BigDecimal, but we want it to be coerced to java Long ; cause it still fits and the tests are expecting that (if (= (.getColumnLabel rsmeta i) "count") (.getLong rs i) (.getBigDecimal rs i)))) (defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/ARRAY] [_ ^ResultSet rs ^ResultSetMetaData _rsmeta ^Integer i] (fn [] (when-let [arr (.getArray rs i)] (Arrays/deepToString (.getArray arr))))) (defn- ipv4-column->string [^ResultSet rs ^Integer i] (when-let [inet-address (.getObject rs i java.net.Inet4Address)] (.getHostAddress inet-address))) (defn- ipv6-column->string [^ResultSet rs ^Integer i] (when-let [inet-address (.getObject rs i java.net.Inet6Address)] (.getHostAddress inet-address))) (defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/OTHER] [_ ^ResultSet rs ^ResultSetMetaData rsmeta ^Integer i] (fn [] (let [normalized-db-type (remove-low-cardinality-and-nullable (.getColumnTypeName rsmeta i))] (cond (= normalized-db-type "ipv4") (ipv4-column->string rs i) (= normalized-db-type "ipv6") (ipv6-column->string rs i) ;; _ :else (.getObject rs i))))) (defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/VARCHAR] [_ ^ResultSet rs ^ResultSetMetaData rsmeta ^Integer i] (fn [] (let [normalized-db-type (remove-low-cardinality-and-nullable (.getColumnTypeName rsmeta i))] (cond ;; Enum8/Enum16 (str/starts-with? normalized-db-type "enum") (.getString rs i) ;; _ :else (.getObject rs i))))) (defmethod unprepare/unprepare-value [:clickhouse LocalDate] [_ t] (format "'%s'" (t/format "yyyy-MM-dd" t))) (defmethod unprepare/unprepare-value [:clickhouse LocalTime] [_ t] (format "'%s'" (t/format "HH:mm:ss.SSS" t))) (defmethod unprepare/unprepare-value [:clickhouse OffsetTime] [_ t] (format "'%s'" (t/format "HH:mm:ss.SSSZZZZZ" t))) (defmethod unprepare/unprepare-value [:clickhouse LocalDateTime] [_ t] (format "'%s'" (t/format "yyyy-MM-dd HH:mm:ss.SSS" t))) (defmethod unprepare/unprepare-value [:clickhouse OffsetDateTime] [_ ^OffsetDateTime t] (format "%s('%s')" (if (zero? (.getNano t)) "parseDateTimeBestEffort" "parseDateTime64BestEffort") (t/format "yyyy-MM-dd HH:mm:ss.SSSZZZZZ" t))) (defmethod unprepare/unprepare-value [:clickhouse ZonedDateTime] [_ t] (format "'%s'" (t/format "yyyy-MM-dd HH:mm:ss.SSSZZZZZ" t))) ================================================ FILE: src/metabase/driver/clickhouse_version.clj ================================================ "Provides the info about the ClickHouse version. Extracted from the main clickhouse.clj file, as both Driver and QP overrides require access to it, avoiding circular dependencies." (ns metabase.driver.clickhouse-version (:require [clojure.core.memoize :as memoize] [metabase.driver :as driver] [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn] [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute] [metabase.driver.util :as driver.u] [metabase.lib.metadata :as lib.metadata] [metabase.query-processor.store :as qp.store])) (set! *warn-on-reflection* true) ;; cache the results for 60 minutes; ;; TTL is here only to eventually clear out old entries/keep it from growing too large (def ^:private default-cache-ttl (* 60 60 1000)) (def ^:private clickhouse-version-query (str "WITH s AS (SELECT version() AS ver, splitByChar('.', ver) AS verSplit) " "SELECT s.ver, toInt32(verSplit[1]), toInt32(verSplit[2]) FROM s")) (def ^:private ^{:arglists '([db-details])} get-clickhouse-version (memoize/ttl (fn [db-details] (sql-jdbc.execute/do-with-connection-with-options :clickhouse (sql-jdbc.conn/connection-details->spec :clickhouse db-details) nil (fn [^java.sql.Connection conn] (with-open [stmt (.createStatement conn) rset (.executeQuery stmt clickhouse-version-query)] (when (.next rset) {:version (.getString rset 1) :semantic-version {:major (.getInt rset 2) :minor (.getInt rset 3)}}))))) :ttl/threshold default-cache-ttl)) (defmethod driver/dbms-version :clickhouse [_driver db] (get-clickhouse-version (:details db))) (defn is-at-least? "Is ClickHouse version at least `major.minor` (e.g., 24.4)?" ([major minor] ;; used from the QP overrides; we don't have access to the DB object (is-at-least? major minor (lib.metadata/database (qp.store/metadata-provider)))) ([major minor db] ;; used from the Driver overrides; we have access to the DB object (let [version (driver/dbms-version :clickhouse db) semantic (:semantic-version version)] (driver.u/semantic-version-gte [(:major semantic) (:minor semantic)] [major minor])))) (defn with-min "Execute `f` if the ClickHouse version is greater or equal to `major.minor` (e.g., 24.4); otherwise, execute `fallback-f`, if it's provided." ([major minor f] (with-min major minor f nil)) ([major minor f fallback-f] (if (is-at-least? major minor) (f) (when (not (nil? fallback-f)) (fallback-f))))) ================================================ FILE: test/metabase/driver/clickhouse_data_types_test.clj ================================================ (ns metabase.driver.clickhouse-data-types-test #_{:clj-kondo/ignore [:unsorted-required-namespaces]} (:require [cljc.java-time.local-date :as local-date] [cljc.java-time.local-date-time :as local-date-time] [clojure.test :refer :all] [metabase.query-processor.test-util :as qp.test] [metabase.test :as mt] [metabase.test.data :as data] [metabase.test.data.clickhouse :as ctd] [metabase.test.data.interface :as tx])) (use-fixtures :once ctd/create-test-db!) (deftest ^:parallel clickhouse-decimals (mt/test-driver :clickhouse (data/dataset (tx/dataset-definition "mbt" ["decimals" [{:field-name "my_money" :base-type {:native "Decimal(12,4)"}}] [[1.0] [23.1337] [42.0] [42.0]]]) (testing "simple division" (is (= 21.0 (-> (data/run-mbql-query decimals {:expressions {:divided [:/ $my_money 2]} :filter [:> [:expression :divided] 1.0] :breakout [[:expression :divided]] :order-by [[:desc [:expression :divided]]] :limit 1}) qp.test/first-row last float)))) (testing "divided decimal precision" (is (= 1.8155331831916208 (-> (data/run-mbql-query decimals {:expressions {:divided [:/ 42 $my_money]} :filter [:= $id 2] :limit 1}) qp.test/first-row last double))))))) #_(deftest ^:parallel clickhouse-array-string (mt/test-driver :clickhouse (is (= "[foo, bar]" (-> (data/dataset (tx/dataset-definition "metabase_tests_array_string" ["test-data-array-string" [{:field-name "my_array" :base-type {:native "Array(String)"}}] [[(into-array (list "foo" "bar"))]]]) (data/run-mbql-query test-data-array-string {:limit 1})) qp.test/first-row last))))) #_(deftest ^:parallel clickhouse-array-uint64 (mt/test-driver :clickhouse (is (= "[23, 42]" (-> (data/dataset (tx/dataset-definition "metabase_tests_array_uint" ["test-data-array-uint64" [{:field-name "my_array" :base-type {:native "Array(UInt64)"}}] [[(into-array (list 23 42))]]]) (data/run-mbql-query test-data-array-uint64 {:limit 1})) qp.test/first-row last))))) #_(deftest ^:parallel clickhouse-array-of-arrays (mt/test-driver :clickhouse (let [row1 (into-array (list (into-array (list "foo" "bar")) (into-array (list "qaz" "qux")))) row2 (into-array nil) query-result (data/dataset (tx/dataset-definition "metabase_tests_array_of_arrays" ["test-data-array-of-arrays" [{:field-name "my_array_of_arrays" :base-type {:native "Array(Array(String))"}}] [[row1] [row2]]]) (data/run-mbql-query test-data-array-of-arrays {})) result (ctd/rows-without-index query-result)] (is (= [["[[foo, bar], [qaz, qux]]"], ["[]"]] result))))) #_(deftest ^:parallel clickhouse-low-cardinality-array (mt/test-driver :clickhouse (let [row1 (into-array (list "foo" "bar")) row2 (into-array nil) query-result (data/dataset (tx/dataset-definition "metabase_tests_low_cardinality_array" ["test-data-low-cardinality-array" [{:field-name "my_low_card_array" :base-type {:native "Array(LowCardinality(String))"}}] [[row1] [row2]]]) (data/run-mbql-query test-data-low-cardinality-array {})) result (ctd/rows-without-index query-result)] (is (= [["[foo, bar]"], ["[]"]] result))))) #_(deftest ^:parallel clickhouse-array-of-nullables (mt/test-driver :clickhouse (let [row1 (into-array (list "foo" nil "bar")) row2 (into-array nil) query-result (data/dataset (tx/dataset-definition "metabase_tests_array_of_nullables" ["test-data-array-of-nullables" [{:field-name "my_array_of_nullables" :base-type {:native "Array(Nullable(String))"}}] [[row1] [row2]]]) (data/run-mbql-query test-data-array-of-nullables {})) result (ctd/rows-without-index query-result)] (is (= [["[foo, null, bar]"], ["[]"]] result))))) #_(deftest ^:parallel clickhouse-array-of-booleans (mt/test-driver :clickhouse (let [row1 (into-array (list true false true)) row2 (into-array nil) query-result (data/dataset (tx/dataset-definition "metabase_tests_array_of_booleans" ["test-data-array-of-booleans" [{:field-name "my_array_of_booleans" :base-type {:native "Array(Boolean)"}}] [[row1] [row2]]]) (data/run-mbql-query test-data-array-of-booleans {})) result (ctd/rows-without-index query-result)] (is (= [["[true, false, true]"], ["[]"]] result))))) #_(deftest ^:parallel clickhouse-array-of-nullable-booleans (mt/test-driver :clickhouse (let [row1 (into-array (list true false nil)) row2 (into-array nil) query-result (data/dataset (tx/dataset-definition "metabase_tests_array_of_nullable_booleans" ["test-data-array-of-booleans" [{:field-name "my_array_of_nullable_booleans" :base-type {:native "Array(Nullable(Boolean))"}}] [[row1] [row2]]]) (data/run-mbql-query test-data-array-of-booleans {})) result (ctd/rows-without-index query-result)] (is (= [["[true, false, null]"], ["[]"]] result))))) #_(deftest ^:parallel clickhouse-array-of-uint8 (mt/test-driver :clickhouse (let [row1 (into-array (list 42 100 2)) row2 (into-array nil) query-result (data/dataset (tx/dataset-definition "metabase_tests_array_of_uint8" ["test-data-array-of-uint8" [{:field-name "my_array_of_uint8" :base-type {:native "Array(UInt8)"}}] [[row1] [row2]]]) (data/run-mbql-query test-data-array-of-uint8 {})) result (ctd/rows-without-index query-result)] (is (= [["[42, 100, 2]"], ["[]"]] result))))) #_(deftest ^:parallel clickhouse-array-of-floats (mt/test-driver :clickhouse (let [row1 (into-array (list 1.2 3.4)) row2 (into-array nil) query-result (data/dataset (tx/dataset-definition "metabase_tests_array_of_floats" ["test-data-array-of-floats" [{:field-name "my_array_of_floats" :base-type {:native "Array(Float64)"}}] [[row1] [row2]]]) (data/run-mbql-query test-data-array-of-floats {})) result (ctd/rows-without-index query-result)] (is (= [["[1.2, 3.4]"], ["[]"]] result))))) ;; NB: timezones in the formatted string are purely cosmetic; it will be fine on the UI #_(deftest ^:parallel clickhouse-array-of-dates (mt/test-driver :clickhouse (let [row1 (into-array (list (local-date/parse "2022-12-06") (local-date/parse "2021-10-19"))) row2 (into-array nil) query-result (data/dataset (tx/dataset-definition "metabase_tests_array_of_dates" ["test-data-array-of-dates" [{:field-name "my_array_of_dates" :base-type {:native "Array(Date)"}}] [[row1] [row2]]]) (data/run-mbql-query test-data-array-of-dates {})) result (ctd/rows-without-index query-result)] (is (= [["[2022-12-06T00:00Z[UTC], 2021-10-19T00:00Z[UTC]]"], ["[]"]] result))))) #_(deftest ^:parallel clickhouse-array-of-date32 (mt/test-driver :clickhouse (let [row1 (into-array (list (local-date/parse "2122-12-06") (local-date/parse "2099-10-19"))) row2 (into-array nil) query-result (data/dataset (tx/dataset-definition "metabase_tests_array_of_date32" ["test-data-array-of-date32" [{:field-name "my_array_of_date32" :base-type {:native "Array(Date32)"}}] [[row1] [row2]]]) (data/run-mbql-query test-data-array-of-date32 {})) result (ctd/rows-without-index query-result)] (is (= [["[2122-12-06T00:00Z[UTC], 2099-10-19T00:00Z[UTC]]"], ["[]"]] result))))) #_(deftest ^:parallel clickhouse-array-of-datetime (mt/test-driver :clickhouse (let [row1 (into-array (list (local-date-time/parse "2022-12-06T18:28:31") (local-date-time/parse "2021-10-19T13:12:44"))) row2 (into-array nil) query-result (data/dataset (tx/dataset-definition "metabase_tests_array_of_datetime" ["test-data-array-of-datetime" [{:field-name "my_array_of_datetime" :base-type {:native "Array(DateTime)"}}] [[row1] [row2]]]) (data/run-mbql-query test-data-array-of-datetime {})) result (ctd/rows-without-index query-result)] (is (= [["[2022-12-06T18:28:31Z[UTC], 2021-10-19T13:12:44Z[UTC]]"], ["[]"]] result))))) #_(deftest ^:parallel clickhouse-array-of-datetime64 (mt/test-driver :clickhouse (let [row1 (into-array (list (local-date-time/parse "2022-12-06T18:28:31.123") (local-date-time/parse "2021-10-19T13:12:44.456"))) row2 (into-array nil) query-result (data/dataset (tx/dataset-definition "metabase_tests_array_of_datetime64" ["test-data-array-of-datetime64" [{:field-name "my_array_of_datetime64" :base-type {:native "Array(DateTime64(3))"}}] [[row1] [row2]]]) (data/run-mbql-query test-data-array-of-datetime64 {})) result (ctd/rows-without-index query-result)] (is (= [["[2022-12-06T18:28:31.123Z[UTC], 2021-10-19T13:12:44.456Z[UTC]]"], ["[]"]] result))))) #_(deftest ^:parallel clickhouse-array-of-decimals (mt/test-driver :clickhouse (let [row1 (into-array (list "12345123.123456789" "78.245")) row2 nil query-result (data/dataset (tx/dataset-definition "metabase_tests_array_of_decimals" ["test-data-array-of-decimals" [{:field-name "my_array_of_decimals" :base-type {:native "Array(Decimal(18, 9))"}}] [[row1] [row2]]]) (data/run-mbql-query test-data-array-of-decimals {})) result (ctd/rows-without-index query-result)] (is (= [["[12345123.123456789, 78.245000000]"], ["[]"]] result))))) #_(deftest ^:parallel clickhouse-array-of-tuples (mt/test-driver :clickhouse (is (= [["[[foobar, 1234], [qaz, 0]]"] ["[]"]] (qp.test/formatted-rows [str] :format-nil-values (ctd/do-with-test-db (fn [db] (data/with-db db (data/run-mbql-query array_of_tuples_test {}))))))))) #_(deftest ^:parallel clickhouse-array-of-uuids (mt/test-driver :clickhouse (let [row1 (into-array (list "2eac427e-7596-11ed-a1eb-0242ac120002" "2eac44f4-7596-11ed-a1eb-0242ac120002")) row2 nil query-result (data/dataset (tx/dataset-definition "metabase_tests_array_of_uuids" ["test-data-array-of-uuids" [{:field-name "my_array_of_uuids" :base-type {:native "Array(UUID)"}}] [[row1] [row2]]]) (data/run-mbql-query test-data-array-of-uuids {})) result (ctd/rows-without-index query-result)] (is (= [["[2eac427e-7596-11ed-a1eb-0242ac120002, 2eac44f4-7596-11ed-a1eb-0242ac120002]"], ["[]"]] result))))) #_(deftest ^:parallel clickhouse-array-inner-types (mt/test-driver :clickhouse (is (= [["[a, b, c]" "[null, d, e]" "[1.0000, 2.0000, 3.0000]" "[4.0000, null, 5.0000]"]] (ctd/do-with-test-db (fn [db] (data/with-db db (->> (data/run-mbql-query arrays_inner_types {}) (mt/formatted-rows [str str str str]))))))))) (deftest ^:parallel clickhouse-nullable-strings (mt/test-driver :clickhouse (data/dataset (tx/dataset-definition "metabase_tests_nullable_strings" ["test-data-nullable-strings" [{:field-name "mystring" :base-type :type/Text}] [["foo"] ["bar"] [" "] [""] [nil]]]) (testing "null strings count" (is (= 2M ;; BigDecimal (-> (data/run-mbql-query test-data-nullable-strings {:filter [:is-null $mystring] :aggregation [:count]}) qp.test/first-row last)))) (testing "nullable strings not null filter" (is (= 3M (-> (data/run-mbql-query test-data-nullable-strings {:filter [:not-null $mystring] :aggregation [:count]}) qp.test/first-row last)))) (testing "filter nullable string by value" (is (= 1M (-> (data/run-mbql-query test-data-nullable-strings {:filter [:= $mystring "foo"] :aggregation [:count]}) qp.test/first-row last))))))) (deftest ^:parallel clickhouse-non-latin-strings (mt/test-driver :clickhouse (testing "basic filtering" (is (= [[1 "Я_1"] [3 "Я_2"] [4 "Я"]] (qp.test/formatted-rows [int str] :format-nil-values (ctd/do-with-test-db (fn [db] (data/with-db db (data/run-mbql-query metabase_test_lowercases {:filter [:contains $mystring "Я"]})))))))) (testing "case-insensitive non-latin filtering" (is (= [[1 "Я_1"] [3 "Я_2"] [4 "Я"] [5 "я"]] (qp.test/formatted-rows [int str] :format-nil-values (ctd/do-with-test-db (fn [db] (data/with-db db (data/run-mbql-query metabase_test_lowercases {:filter [:contains $mystring "Я" {:case-sensitive false}]})))))))))) (deftest ^:parallel clickhouse-datetime64-filter (mt/test-driver :clickhouse (let [row1 "2022-03-03 03:03:03.333" row2 "2022-03-03 03:03:03.444" row3 "2022-03-03 03:03:03" query-result (data/dataset (tx/dataset-definition "metabase_tests_datetime64" ["test-data-datetime64" [{:field-name "milli_sec" :base-type {:native "DateTime64(3)"}}] [[row1] [row2] [row3]]]) (data/run-mbql-query test-data-datetime64 {:filter [:= $milli_sec "2022-03-03T03:03:03.333Z"]})) result (ctd/rows-without-index query-result)] (is (= [["2022-03-03T03:03:03.333Z"]] result))))) (deftest ^:parallel clickhouse-datetime-filter (mt/test-driver :clickhouse (let [row1 "2022-03-03 03:03:03" row2 "2022-03-03 03:03:04" row3 "2022-03-03 03:03:05" query-result (data/dataset (tx/dataset-definition "metabase_tests_datetime" ["test-data-datetime" [{:field-name "second" :base-type {:native "DateTime"}}] [[row1] [row2] [row3]]]) (data/run-mbql-query test-data-datetime {:filter [:= $second "2022-03-03T03:03:04Z"]})) result (ctd/rows-without-index query-result)] (is (= [["2022-03-03T03:03:04Z"]] result))))) (deftest ^:parallel clickhouse-booleans (mt/test-driver :clickhouse (let [[row1 row2 row3 row4] [["#1" true] ["#2" false] ["#3" false] ["#4" true]] query-result (data/dataset (tx/dataset-definition "metabase_tests_booleans" ["test-data-booleans" [{:field-name "name" :base-type :type/Text} {:field-name "is_active" :base-type :type/Boolean}] [row1 row2 row3 row4]]) (data/run-mbql-query test-data-booleans {:filter [:= $is_active false]})) rows (qp.test/rows query-result) result (map #(drop 1 %) rows)] ; remove db "index" which is the first column in the result set (is (= [row2 row3] result))))) (deftest ^:parallel clickhouse-enums-values-test (mt/test-driver :clickhouse (testing "select enums values as strings" (is (= [["foo" "house" "qaz"] ["foo bar" "click" "qux"] ["bar" "house" "qaz"]] (qp.test/formatted-rows [str str str] :format-nil-values (ctd/do-with-test-db (fn [db] (data/with-db db (data/run-mbql-query enums_test {})))))))) (testing "filtering enum values" (is (= [["useqa"]] (qp.test/formatted-rows [str] :format-nil-values (ctd/do-with-test-db (fn [db] (data/with-db db (data/run-mbql-query enums_test {:expressions {"test" [:concat [:substring $enum2 3 3] [:substring $enum3 1 2]]} :fields [[:expression "test"]] :filter [:= $enum1 "foo"]})))))))))) (deftest ^:parallel clickhouse-ipv4query-test (mt/test-driver :clickhouse (is (= [[1]] (qp.test/formatted-rows [int] :format-nil-values (ctd/do-with-test-db (fn [db] (data/with-db db (data/run-mbql-query ipaddress_test {:filter [:= $ipvfour "127.0.0.1"] :aggregation [[:count]]}))))))))) (deftest ^:parallel clickhouse-ip-serialization-test (mt/test-driver :clickhouse (is (= [["127.0.0.1" "0:0:0:0:0:0:0:1"] ["0.0.0.0" "2001:438:ffff:0:0:0:407d:1bc1"] [nil nil]] (qp.test/formatted-rows [str str] (ctd/do-with-test-db (fn [db] (data/with-db db (data/run-mbql-query ipaddress_test {}))))))))) (defn- map-as-string [^java.util.LinkedHashMap m] (.toString m)) (deftest ^:parallel clickhouse-simple-map-test (mt/test-driver :clickhouse (is (= [["{key1=1, key2=10}"] ["{key1=2, key2=20}"] ["{key1=3, key2=30}"]] (qp.test/formatted-rows [map-as-string] :format-nil-values (ctd/do-with-test-db (fn [db] (data/with-db db (data/run-mbql-query maps_test {}))))))))) (deftest ^:parallel clickhouse-datetime-diff-nullable (mt/test-driver :clickhouse (is (= [[170 202] [nil nil] [nil nil] [nil nil]] (ctd/do-with-test-db (fn [db] (data/with-db db (->> (data/run-mbql-query datetime_diff_nullable {:fields [[:expression "dt64,dt"] [:expression "dt64,d"]] :expressions {"dt64,dt" [:datetime-diff $dt64 $dt :day] "dt64,d" [:datetime-diff $dt64 $d :day]}}) (mt/formatted-rows [int int]))))))))) ;; Metabase has pretty extensive testing for sum-where and count-where ;; However, this ClickHouse-specific corner case is not covered (deftest ^:parallel clickhouse-sum-where-numeric-types (mt/test-driver :clickhouse (testing "int values (with matching rows)" (is (= [[8]] (qp.test/formatted-rows [int] :format-nil-values (ctd/do-with-test-db (fn [db] (data/with-db db (data/run-mbql-query sum_if_test_int {:aggregation [[:sum-where $int_value [:= $discriminator "bar"]]]})))))))) (testing "int values (no matching rows)" (is (= [[0]] (qp.test/formatted-rows [int] :format-nil-values (ctd/do-with-test-db (fn [db] (data/with-db db (data/run-mbql-query sum_if_test_int {:aggregation [[:sum-where $int_value [:= $discriminator "qaz"]]]})))))))) (testing "double values (with matching rows)" (is (= [[9.27]] (qp.test/formatted-rows [double] :format-nil-values (ctd/do-with-test-db (fn [db] (data/with-db db (data/run-mbql-query sum_if_test_float {:aggregation [[:sum-where $float_value [:= $discriminator "bar"]]]})))))))) (testing "double values (no matching rows)" (is (= [[0.0]] (qp.test/formatted-rows [double] :format-nil-values (ctd/do-with-test-db (fn [db] (data/with-db db (data/run-mbql-query sum_if_test_float {:aggregation [[:sum-where $float_value [:= $discriminator "qaz"]]]})))))))))) (deftest ^:parallel clickhouse-unsigned-integers (mt/test-driver :clickhouse (is (= [["255" "65535" "4294967295" "18446744073709551615"]] (qp.test/formatted-rows [str str str str] :format-nil-values (ctd/do-with-test-db (fn [db] (data/with-db db (data/run-mbql-query unsigned_int_types {}))))))))) ;; FIXME: blocked by https://github.com/ClickHouse/clickhouse-java/issues/2218 #_(deftest ^:parallel clickhouse-fixed-strings (mt/test-driver :clickhouse (is (= [["val1" "val2" "val3" "val4"]] (qp.test/formatted-rows [str str str str] :format-nil-values (ctd/do-with-test-db (fn [db] (data/with-db db (data/run-mbql-query fixed_strings {}))))))))) ================================================ FILE: test/metabase/driver/clickhouse_impersonation_test.clj ================================================ (ns metabase.driver.clickhouse-impersonation-test "SET ROLE (connection impersonation feature) tests on with single node or on-premise cluster setups." #_{:clj-kondo/ignore [:unsorted-required-namespaces]} (:require [clojure.test :refer :all] [metabase-enterprise.advanced-permissions.api.util-test :as advanced-perms.api.tu] [metabase.driver :as driver] [metabase.driver.sql :as driver.sql] [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn] [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute] [metabase.query-processor.store :as qp.store] [metabase.test :as mt] [metabase.test.data.clickhouse :as ctd] [metabase.util :as u] [toucan2.tools.with-temp :as t2.with-temp])) ;; 53+ metabase.sync has moved to metabase.sync.core (try (require '[metabase.sync :as sync]) (catch java.io.FileNotFoundException e (when (re-find #"metabase/sync\.clj" (.getMessage e)) (require '[metabase.sync.core :as sync])))) (set! *warn-on-reflection* true) (defn- set-role-test! [details-map] (let [default-role (driver.sql/default-database-role :clickhouse nil) spec (sql-jdbc.conn/connection-details->spec :clickhouse details-map)] (testing "default role is NONE" (is (= default-role "NONE"))) (testing "does not throw with an existing role" (sql-jdbc.execute/do-with-connection-with-options :clickhouse spec nil (fn [^java.sql.Connection conn] (driver/set-role! :clickhouse conn "metabase_test_role"))) (is true)) (testing "does not throw with a role containing hyphens" (sql-jdbc.execute/do-with-connection-with-options :clickhouse spec nil (fn [^java.sql.Connection conn] (driver/set-role! :clickhouse conn "metabase-test-role"))) (is true)) (testing "does not throw with the default role" (sql-jdbc.execute/do-with-connection-with-options :clickhouse spec nil (fn [^java.sql.Connection conn] (driver/set-role! :clickhouse conn default-role) (fn [^java.sql.Connection conn] (driver/set-role! :clickhouse conn default-role) (with-open [stmt (.prepareStatement conn "SELECT * FROM `metabase_test_role_db`.`some_table` ORDER BY i ASC;") rset (.executeQuery stmt)] (is (.next rset) true) (is (.getInt rset 1) 42) (is (.next rset) true) (is (.getInt rset 1) 144) (is (.next rset) false))))) (is true)))) (defn- set-role-throws-test! [details-map] (testing "throws when assigning a non-existent role" (is (thrown? Exception (sql-jdbc.execute/do-with-connection-with-options :clickhouse (sql-jdbc.conn/connection-details->spec :clickhouse details-map) nil (fn [^java.sql.Connection conn] (driver/set-role! :clickhouse conn "asdf"))))))) (defn- do-with-new-metadata-provider [details thunk] (t2.with-temp/with-temp [:model/Database db {:engine :clickhouse :details details}] (qp.store/with-metadata-provider (u/the-id db) (thunk db)))) (deftest clickhouse-set-role (mt/test-driver :clickhouse (let [user-details {:user "metabase_test_user"} ;; See docker-compose.yml for the port mappings ;; 24.4+ single-node-port-details {:port 8123} single-node-details (merge user-details single-node-port-details) cluster-port-details {:port 8127} cluster-details (merge user-details cluster-port-details)] (testing "single node" (testing "should support the impersonation feature" (t2.with-temp/with-temp [:model/Database db {:engine :clickhouse :details {:user "default" :port 8123}}] (is (true? (driver/database-supports? :clickhouse :connection-impersonation db))))) (let [statements ["CREATE DATABASE IF NOT EXISTS `metabase_test_role_db`;" "CREATE OR REPLACE TABLE `metabase_test_role_db`.`some_table` (i Int32) ENGINE = MergeTree ORDER BY (i);" "INSERT INTO `metabase_test_role_db`.`some_table` VALUES (42), (144);" "CREATE ROLE IF NOT EXISTS `metabase_test_role`;" "CREATE ROLE IF NOT EXISTS `metabase-test-role`;" "CREATE USER IF NOT EXISTS `metabase_test_user` NOT IDENTIFIED;" "GRANT SELECT ON `metabase_test_role_db`.* TO `metabase_test_role`,`metabase-test-role`;" "GRANT `metabase_test_role`, `metabase-test-role` TO `metabase_test_user`;"]] (ctd/exec-statements statements single-node-port-details) (do-with-new-metadata-provider single-node-details (fn [_db] (set-role-test! single-node-details) (set-role-throws-test! single-node-details))))) (testing "on-premise cluster" (testing "should support the impersonation feature" (t2.with-temp/with-temp [:model/Database db {:engine :clickhouse :details {:user "default" :port 8127}}] (is (true? (driver/database-supports? :clickhouse :connection-impersonation db))))) (let [statements ["CREATE DATABASE IF NOT EXISTS `metabase_test_role_db` ON CLUSTER '{cluster}';" "CREATE OR REPLACE TABLE `metabase_test_role_db`.`some_table` ON CLUSTER '{cluster}' (i Int32) ENGINE ReplicatedMergeTree('/clickhouse/{cluster}/tables/{database}/{table}/{shard}', '{replica}') ORDER BY (i);" "INSERT INTO `metabase_test_role_db`.`some_table` VALUES (42), (144);" "CREATE ROLE IF NOT EXISTS `metabase_test_role` ON CLUSTER '{cluster}';" "CREATE ROLE IF NOT EXISTS `metabase-test-role` ON CLUSTER '{cluster}';" "CREATE USER IF NOT EXISTS `metabase_test_user` ON CLUSTER '{cluster}' NOT IDENTIFIED;" "GRANT ON CLUSTER '{cluster}' SELECT ON `metabase_test_role_db`.* TO `metabase_test_role`, `metabase-test-role`;" "GRANT ON CLUSTER '{cluster}' `metabase_test_role`, `metabase-test-role` TO `metabase_test_user`;"]] (ctd/exec-statements statements cluster-port-details) (do-with-new-metadata-provider cluster-details (fn [_db] (set-role-test! cluster-details) (set-role-throws-test! cluster-details))))) (testing "older ClickHouse version" ;; 23.3 (testing "should NOT support the impersonation feature" (t2.with-temp/with-temp [:model/Database db {:engine :clickhouse :details {:user "default" :port 8124}}] (is (false? (driver/database-supports? :clickhouse :connection-impersonation db))))))))) (deftest conn-impersonation-test-clickhouse (mt/test-driver :clickhouse (mt/with-premium-features #{:advanced-permissions} (let [table-name (str "metabase_impersonation_test.test_" (System/currentTimeMillis)) select-query (format "SELECT * FROM %s;" table-name) cluster-port {:port 8127} cluster-details {:engine :clickhouse :details {:user "metabase_impersonation_test_user" :dbname "metabase_impersonation_test" :port 8127}} ddl-statements ["CREATE DATABASE IF NOT EXISTS metabase_impersonation_test ON CLUSTER '{cluster}';" (format "CREATE TABLE %s ON CLUSTER '{cluster}' (s String) ENGINE ReplicatedMergeTree('/clickhouse/{cluster}/tables/{database}/{table}/{shard}', '{replica}') ORDER BY (s);" table-name)] insert-statements [(format "INSERT INTO %s VALUES ('a'), ('b'), ('c');" table-name)] grant-statements ["CREATE USER IF NOT EXISTS metabase_impersonation_test_user ON CLUSTER '{cluster}' NOT IDENTIFIED;" "CREATE ROLE IF NOT EXISTS row_a ON CLUSTER '{cluster}';" "CREATE ROLE IF NOT EXISTS row_b ON CLUSTER '{cluster}';" "CREATE ROLE IF NOT EXISTS row_c ON CLUSTER '{cluster}';" "GRANT ON CLUSTER '{cluster}' row_a, row_b, row_c TO metabase_impersonation_test_user;" (format "GRANT ON CLUSTER '{cluster}' SELECT ON %s TO metabase_impersonation_test_user;" table-name) (format "CREATE ROW POLICY OR REPLACE policy_row_a ON CLUSTER '{cluster}' ON %s FOR SELECT USING s = 'a' TO row_a;" table-name) (format "CREATE ROW POLICY OR REPLACE policy_row_b ON CLUSTER '{cluster}' ON %s FOR SELECT USING s = 'b' TO row_b;" table-name) (format "CREATE ROW POLICY OR REPLACE policy_row_c ON CLUSTER '{cluster}' ON %s FOR SELECT USING s = 'c' TO row_c;" table-name)]] (ctd/exec-statements ddl-statements cluster-port {"wait_end_of_query" "1"}) (ctd/exec-statements insert-statements cluster-port {"wait_end_of_query" "1" "insert_quorum" "2"}) (ctd/exec-statements grant-statements cluster-port {"wait_end_of_query" "1"}) (t2.with-temp/with-temp [:model/Database db cluster-details] (mt/with-db db (sync/sync-database! db) (letfn [(check-impersonation! [roles expected] (advanced-perms.api.tu/with-impersonations! {:impersonations [{:db-id (mt/id) :attribute "impersonation_attr"}] :attributes {"impersonation_attr" roles}} (is (= expected (-> {:query select-query} mt/native-query mt/process-query mt/rows)))))] (is (= [["a"] ["b"] ["c"]] (-> {:query select-query} mt/native-query mt/process-query mt/rows))) (check-impersonation! "row_a" [["a"]]) (check-impersonation! "row_b" [["b"]]) (check-impersonation! "row_c" [["c"]]) (check-impersonation! "row_a,row_c" [["a"] ["c"]]) (check-impersonation! "row_b,row_c" [["b"] ["c"]]) (check-impersonation! "row_a,row_b,row_c" [["a"] ["b"] ["c"]])))))))) ================================================ FILE: test/metabase/driver/clickhouse_introspection_test.clj ================================================ (ns metabase.driver.clickhouse-introspection-test #_{:clj-kondo/ignore [:unsorted-required-namespaces]} (:require [clojure.test :refer :all] [metabase.driver :as driver] [metabase.driver.common :as driver.common] [metabase.query-processor :as qp] [metabase.query-processor.test-util :as qp.test] [metabase.test :as mt] [metabase.test.data :as data] [metabase.test.data.clickhouse :as ctd] [metabase.test.data.interface :as tx] [toucan2.tools.with-temp :as t2.with-temp])) (use-fixtures :once ctd/create-test-db!) (defn- desc-table [table-name] (into #{} (map #(select-keys % [:name :database-type :base-type :database-required]) (:fields (ctd/do-with-test-db #(driver/describe-table :clickhouse % {:name table-name})))))) (deftest ^:parallel clickhouse-base-types-test-enums (mt/test-driver :clickhouse (testing "enums" (let [table-name "enums_base_types"] (is (= #{{:base-type :type/Text, :database-required false, :database-type "Nullable(Enum8('America/New_York' = 1))", :name "c1"} {:base-type :type/Text, :database-required true, :database-type "Enum8('BASE TABLE' = 1, 'VIEW' = 2, 'FOREIGN TABLE' = 3, 'LOCAL TEMPORARY' = 4, 'SYSTEM VIEW' = 5)", :name "c2"} {:base-type :type/Text, :database-required true, :database-type "Enum8('NO' = 1, 'YES' = 2)", :name "c3"} {:base-type :type/Text, :database-required true, :database-type "Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2)", :name "c4"} {:base-type :type/Text, :database-required false, :database-type "Nullable(Enum8('GLOBAL' = 0, 'DATABASE' = 1, 'TABLE' = 2))", :name "c5"} {:base-type :type/Text, :database-required false, :database-type "Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2))", :name "c6"}} (desc-table table-name))))))) (deftest ^:parallel clickhouse-base-types-test-dates (mt/test-driver :clickhouse (testing "dates" (let [table-name "date_base_types"] (is (= #{{:base-type :type/Date, :database-required true, :database-type "Date", :name "c1"} {:base-type :type/Date, :database-required true, :database-type "Date32", :name "c2"} {:base-type :type/Date, :database-required false, :database-type "Nullable(Date)", :name "c3"} {:base-type :type/Date, :database-required false, :database-type "Nullable(Date32)", :name "c4"}} (desc-table table-name))))))) (deftest ^:parallel clickhouse-base-types-test-datetimes (mt/test-driver :clickhouse (testing "datetimes" (let [table-name "datetime_base_types"] (is (= #{{:base-type :type/DateTimeWithLocalTZ, :database-required false, :database-type "Nullable(DateTime('America/New_York'))", :name "c1"} {:base-type :type/DateTimeWithLocalTZ, :database-required true, :database-type "DateTime('America/New_York')", :name "c2"} {:base-type :type/DateTimeWithLocalTZ, :database-required true, :database-type "DateTime", :name "c3"} {:base-type :type/DateTimeWithLocalTZ, :database-required true, :database-type "DateTime64(3)", :name "c4"} {:base-type :type/DateTimeWithLocalTZ, :database-required true, :database-type "DateTime64(9, 'America/New_York')", :name "c5"} {:base-type :type/DateTimeWithLocalTZ, :database-required false, :database-type "Nullable(DateTime64(6, 'America/New_York'))", :name "c6"} {:base-type :type/DateTimeWithLocalTZ, :database-required false, :database-type "Nullable(DateTime64(0))", :name "c7"} {:base-type :type/DateTimeWithLocalTZ, :database-required false, :database-type "Nullable(DateTime)", :name "c8"}} (desc-table table-name))))))) (deftest ^:parallel clickhouse-base-types-test-integers (mt/test-driver :clickhouse (testing "integers" (let [table-name "integer_base_types"] (is (= #{{:base-type :type/Integer, :database-required true, :database-type "UInt8", :name "c1"} {:base-type :type/Integer, :database-required true, :database-type "UInt16", :name "c2"} {:base-type :type/Integer, :database-required true, :database-type "UInt32", :name "c3"} {:base-type :type/BigInteger, :database-required true, :database-type "UInt64", :name "c4"} {:base-type :type/*, :database-required true, :database-type "UInt128", :name "c5"} {:base-type :type/*, :database-required true, :database-type "UInt256", :name "c6"} {:base-type :type/Integer, :database-required true, :database-type "Int8", :name "c7"} {:base-type :type/Integer, :database-required true, :database-type "Int16", :name "c8"} {:base-type :type/Integer, :database-required true, :database-type "Int32", :name "c9"} {:base-type :type/BigInteger, :database-required true, :database-type "Int64", :name "c10"} {:base-type :type/*, :database-required true, :database-type "Int128", :name "c11"} {:base-type :type/*, :database-required true, :database-type "Int256", :name "c12"} {:base-type :type/Integer, :database-required false, :database-type "Nullable(Int32)", :name "c13"}} (desc-table table-name))))))) (deftest ^:parallel clickhouse-base-types-test-numerics (mt/test-driver :clickhouse (testing "numerics" (let [table-name "numeric_base_types"] (is (= #{{:base-type :type/Float, :database-required true, :database-type "Float32", :name "c1"} {:base-type :type/Float, :database-required true, :database-type "Float64", :name "c2"} {:base-type :type/Decimal, :database-required true, :database-type "Decimal(4, 2)", :name "c3"} {:base-type :type/Decimal, :database-required true, :database-type "Decimal(9, 7)", :name "c4"} {:base-type :type/Decimal, :database-required true, :database-type "Decimal(18, 12)", :name "c5"} {:base-type :type/Decimal, :database-required true, :database-type "Decimal(38, 24)", :name "c6"} {:base-type :type/Decimal, :database-required true, :database-type "Decimal(76, 42)", :name "c7"} {:base-type :type/Float, :database-required false, :database-type "Nullable(Float32)", :name "c8"} {:base-type :type/Decimal, :database-required false, :database-type "Nullable(Decimal(4, 2))", :name "c9"} {:base-type :type/Decimal, :database-required false, :database-type "Nullable(Decimal(76, 42))", :name "c10"}} (desc-table table-name))))))) (deftest ^:parallel clickhouse-base-types-test-strings (mt/test-driver :clickhouse (testing "strings" (let [table-name "string_base_types"] (is (= #{{:base-type :type/Text, :database-required true, :database-type "String", :name "c1"} {:base-type :type/Text, :database-required true, :database-type "LowCardinality(String)", :name "c2"} {:base-type :type/TextLike, :database-required true, :database-type "FixedString(32)", :name "c3"} {:base-type :type/Text, :database-required false, :database-type "Nullable(String)", :name "c4"} {:base-type :type/TextLike, :database-required true, :database-type "LowCardinality(FixedString(4))", :name "c5"}} (desc-table table-name))))))) (deftest ^:parallel clickhouse-base-types-test-arrays (mt/test-driver :clickhouse (testing "arrays" (let [table-name "array_base_types"] (is (= #{{:base-type :type/Array, :database-required true, :database-type "Array(String)", :name "c1"} {:base-type :type/Array, :database-required true, :database-type "Array(Nullable(Int32))", :name "c2"} {:base-type :type/Array, :database-required true, :database-type "Array(Array(LowCardinality(FixedString(32))))", :name "c3"} {:base-type :type/Array, :database-required true, :database-type "Array(Array(Array(String)))", :name "c4"}} (desc-table table-name))))))) (deftest ^:parallel clickhouse-base-types-test-low-cardinality-nullable (mt/test-driver :clickhouse (testing "low cardinality nullable" (let [table-name "low_cardinality_nullable_base_types"] (is (= #{{:base-type :type/Text, :database-required true, :database-type "LowCardinality(Nullable(String))", :name "c1"} {:base-type :type/TextLike, :database-required true, :database-type "LowCardinality(Nullable(FixedString(16)))", :name "c2"}} (desc-table table-name))))))) (deftest ^:parallel clickhouse-base-types-test-misc (mt/test-driver :clickhouse (testing "everything else" (let [table-name "misc_base_types"] (is (= #{{:base-type :type/Boolean, :database-required true, :database-type "Bool", :name "c1"} {:base-type :type/UUID, :database-required true, :database-type "UUID", :name "c2"} {:base-type :type/IPAddress, :database-required true, :database-type "IPv4", :name "c3"} {:base-type :type/IPAddress, :database-required true, :database-type "IPv6", :name "c4"} {:base-type :type/Dictionary, :database-required true, :database-type "Map(Int32, String)", :name "c5"} {:base-type :type/Boolean, :database-required false, :database-type "Nullable(Bool)", :name "c6"} {:base-type :type/UUID, :database-required false, :database-type "Nullable(UUID)", :name "c7"} {:base-type :type/IPAddress, :database-required false, :database-type "Nullable(IPv4)", :name "c8"} {:base-type :type/IPAddress, :database-required false, :database-type "Nullable(IPv6)", :name "c9"} {:base-type :type/*, :database-required true, :database-type "Tuple(String, Int32)", :name "c10"}} (desc-table table-name))))))) (deftest ^:parallel clickhouse-boolean-type-metadata (mt/test-driver :clickhouse (let [result (-> {:query "SELECT false, 123, true"} mt/native-query qp/process-query) [[c1 _ c3]] (-> result qp.test/rows)] (testing "column should be of type :type/Boolean" (is (= :type/Boolean (-> result :data :results_metadata :columns first :base_type))) (is (= :type/Boolean (transduce identity (driver.common/values->base-type) [c1, c3]))) (is (= :type/Boolean (driver.common/class->base-type (class c1)))))))) (def ^:private base-field {:database-is-auto-increment false :json-unfolding false :database-required true}) (deftest ^:parallel clickhouse-filtered-aggregate-functions-test-table-metadata (mt/test-driver :clickhouse (is (= {:name "aggregate_functions_filter_test" :fields #{(merge base-field {:name "idx" :database-type "UInt8" :base-type :type/Integer :database-position 0}) (merge base-field {:name "lowest_value" :database-type "SimpleAggregateFunction(min, UInt8)" :base-type :type/Integer :database-position 2}) (merge base-field {:name "count" :database-type "SimpleAggregateFunction(sum, Int64)" :base-type :type/BigInteger :database-position 3})}} (ctd/do-with-test-db (fn [db] (driver/describe-table :clickhouse db {:name "aggregate_functions_filter_test"}))))))) (deftest ^:parallel clickhouse-filtered-aggregate-functions-test-result-set (mt/test-driver :clickhouse (is (= [[42 144 255255]] (qp.test/formatted-rows [int int int] :format-nil-values (ctd/do-with-test-db (fn [db] (data/with-db db (data/run-mbql-query aggregate_functions_filter_test {}))))))))) (def ^:private test-tables #{{:description nil, :name "table1", :schema "metabase_db_scan_test"} {:description nil, :name "table2", :schema "metabase_db_scan_test"}}) (deftest ^:parallel clickhouse-describe-database-single (mt/test-driver :clickhouse (t2.with-temp/with-temp [:model/Database db {:engine :clickhouse :details (merge {:scan-all-databases nil} (tx/dbdef->connection-details :clickhouse :db {:database-name "metabase_db_scan_test"}))}] (let [describe-result (driver/describe-database :clickhouse db)] (is (= {:tables test-tables} describe-result)))))) (deftest ^:parallel clickhouse-describe-database-all (mt/test-driver :clickhouse (t2.with-temp/with-temp [:model/Database db {:engine :clickhouse :details (merge {:scan-all-databases true} (tx/dbdef->connection-details :clickhouse :db {:database-name "default"}))}] (let [describe-result (driver/describe-database :clickhouse db)] ;; check the existence of at least some test tables here (doseq [table test-tables] (is (contains? (:tables describe-result) table))) ;; should not contain any ClickHouse system tables (is (not (some #(= (:schema %) "system") (:tables describe-result)))) (is (not (some #(= (:schema %) "information_schema") (:tables describe-result)))) (is (not (some #(= (:schema %) "INFORMATION_SCHEMA") (:tables describe-result)))))))) (deftest ^:parallel clickhouse-describe-database-multiple (mt/test-driver :clickhouse (t2.with-temp/with-temp [:model/Database db {:engine :clickhouse :details (tx/dbdef->connection-details :clickhouse :db {:database-name "metabase_db_scan_test information_schema"})}] (let [{:keys [tables] :as _describe-result} (driver/describe-database :clickhouse db) tables-table {:name "tables" :description nil :schema "information_schema"} columns-table {:name "columns" :description nil :schema "information_schema"}] ;; tables from `metabase_db_scan_test` (doseq [table test-tables] (is (contains? tables table))) ;; tables from `information_schema` (is (contains? tables tables-table)) (is (contains? tables columns-table)))))) ================================================ FILE: test/metabase/driver/clickhouse_substitution_test.clj ================================================ (ns metabase.driver.clickhouse-substitution-test #_{:clj-kondo/ignore [:unsorted-required-namespaces]} (:require [clojure.test :refer :all] [java-time.api :as t] [metabase.query-processor :as qp] [metabase.test :as mt] [metabase.test.data :as data] [metabase.test.data.interface :as tx] [metabase.test.data.clickhouse :as ctd] [metabase.util :as u] [schema.core :as s]) (:import (java.time LocalDate LocalDateTime))) (set! *warn-on-reflection* true) (defn- get-mbql [value db] (let [uuid (str (java.util.UUID/randomUUID))] {:database (mt/id) :type "native" :native {:collection "test-table" :template-tags {:x {:id uuid :name "d" :display-name "D" :type "dimension" :dimension ["field" (mt/id :test-table :d) nil] :required true}} :query (format "SELECT * FROM `%s`.`test_table` WHERE {{x}}" db)} :parameters [{:type "date/all-options" :value value :target ["dimension" ["template-tag" "x"]] :id uuid}]})) (def ^:private clock (t/mock-clock (t/instant "2019-11-30T23:00:00Z") (t/zone-id "UTC"))) (s/defn ^:private local-date-now :- LocalDate [] (LocalDate/now clock)) (s/defn ^:private local-date-time-now :- LocalDateTime [] (LocalDateTime/now clock)) (deftest ^:parallel clickhouse-variables-field-filters-datetime-and-datetime64 (mt/test-driver :clickhouse (mt/with-clock clock (letfn [(->clickhouse-input [^LocalDateTime ldt] [(t/format "yyyy-MM-dd HH:mm:ss" ldt)]) (get-test-table [rows native-type] ["test_table" [{:field-name "d" :base-type {:native native-type}}] (map ->clickhouse-input rows)]) (->iso-str [^LocalDateTime ldt] (t/format "yyyy-MM-dd'T'HH:mm:ss'Z'" ldt))] (doseq [base-type ["DateTime" "DateTime64"]] (testing base-type (testing "on specific" (let [db (format "mb_vars_on_x_%s" (u/lower-case-en base-type)) now (local-date-time-now) row1 (.minusHours now 14) row2 (.minusMinutes now 20) row3 (.plusMinutes now 5) row4 (.plusHours now 6) table (get-test-table [row1 row2 row3 row4] base-type)] (data/dataset (tx/dataset-definition db table) (testing "date" (is (= [[(->iso-str row1)] [(->iso-str row2)] [(->iso-str row3)]] (ctd/rows-without-index (qp/process-query (get-mbql "2019-11-30" db)))))) (testing "datetime" (is (= [[(->iso-str row2)]] (ctd/rows-without-index (qp/process-query (get-mbql "2019-11-30T22:40:00" db))))))))) (testing "past/next minutes" (let [db (format "mb_vars_m_%s" (u/lower-case-en base-type)) now (local-date-time-now) row1 (.minusHours now 14) row2 (.minusMinutes now 20) row3 (.plusMinutes now 5) row4 (.plusHours now 6) table (get-test-table [row1 row2 row3 row4] base-type)] (data/dataset (tx/dataset-definition db table) (testing "past30minutes" (is (= [[(->iso-str row2)]] (ctd/rows-without-index (qp/process-query (get-mbql "past30minutes" db)))))) (testing "next30minutes" (is (= [[(->iso-str row3)]] (ctd/rows-without-index (qp/process-query (get-mbql "next30minutes" db))))))))) (testing "past/next hours" (let [db (format "mb_vars__past_next_hours_%s" (u/lower-case-en base-type)) now (local-date-time-now) row1 (.minusHours now 14) row2 (.minusHours now 2) row3 (.plusHours now 25) row4 (.plusHours now 6) table (get-test-table [row1 row2 row3 row4] base-type)] (data/dataset (tx/dataset-definition db table) (testing "past12hours" (is (= [[(->iso-str row2)]] (ctd/rows-without-index (qp/process-query (get-mbql "past12hours" db)))))) (testing "next12hours" (is (= [[(->iso-str row4)]] (ctd/rows-without-index (qp/process-query (get-mbql "next12hours" db))))))))) (testing "past/next days" (let [db (format "mb_vars_d_%s" (u/lower-case-en base-type)) now (local-date-time-now) row1 (.minusDays now 14) row2 (.minusDays now 2) row3 (.plusDays now 25) row4 (.plusDays now 6) table (get-test-table [row1 row2 row3 row4] base-type)] (data/dataset (tx/dataset-definition db table) (testing "past12days" (is (= [[(->iso-str row2)]] (ctd/rows-without-index (qp/process-query (get-mbql "past12days" db)))))) (testing "next12days" (is (= [[(->iso-str row4)]] (ctd/rows-without-index (qp/process-query (get-mbql "next12days" db))))))))) (testing "past/next months/quarters" (let [db (format "mb_vars_m_q_%s" (u/lower-case-en base-type)) now (local-date-time-now) row1 (.minusMonths now 14) row2 (.minusMonths now 4) row3 (.plusMonths now 25) row4 (.plusMonths now 6) table (get-test-table [row1 row2 row3 row4] base-type)] (data/dataset (tx/dataset-definition db table) (testing "past12months" (is (= [[(->iso-str row2)]] (ctd/rows-without-index (qp/process-query (get-mbql "past12months" db)))))) (testing "next12months" (is (= [[(->iso-str row4)]] (ctd/rows-without-index (qp/process-query (get-mbql "next12months" db)))))) (testing "past3quarters" (is (= [[(->iso-str row2)]] (ctd/rows-without-index (qp/process-query (get-mbql "past3quarters" db)))))) (testing "next3quarters" (is (= [[(->iso-str row4)]] (ctd/rows-without-index (qp/process-query (get-mbql "next3quarters" db))))))))) (testing "past/next years" (let [db (format "mb_vars_y_%s" (u/lower-case-en base-type)) now (local-date-time-now) row1 (.minusYears now 14) row2 (.minusYears now 4) row3 (.plusYears now 25) row4 (.plusYears now 6) table (get-test-table [row1 row2 row3 row4] base-type)] (data/dataset (tx/dataset-definition db table) (testing "past12years" (is (= [[(->iso-str row2)]] (ctd/rows-without-index (qp/process-query (get-mbql "past12years" db)))))) (testing "next12years" (is (= [[(->iso-str row4)]] (ctd/rows-without-index (qp/process-query (get-mbql "next12years" db))))))))))))))) (deftest ^:parallel clickhouse-variables-field-filters-date-and-date32 (mt/test-driver :clickhouse (mt/with-clock clock (letfn [(->clickhouse-input [^LocalDate ld] [(t/format "yyyy-MM-dd" ld)]) (get-test-table [rows native-type] ["test_table" [{:field-name "d" :base-type {:native native-type}}] (map ->clickhouse-input rows)]) (->iso-str [^LocalDate ld] (str (t/format "yyyy-MM-dd" ld) "T00:00:00Z"))] (doseq [base-type ["Date" "Date32"]] (testing base-type (testing "on specific date" (let [db (format "mb_vars_on_x_%s" (u/lower-case-en base-type)) now (local-date-time-now) row1 (.minusDays now 14) row2 now row3 (.plusDays now 25) row4 (.plusDays now 6) table (get-test-table [row1 row2 row3 row4] base-type)] (data/dataset (tx/dataset-definition db table) (is (= [[(->iso-str row2)]] (ctd/rows-without-index (qp/process-query (get-mbql "2019-11-30" db)))))))) (testing "past/next days" (let [db (format "mb_vars_d_%s" (u/lower-case-en base-type)) now (local-date-now) row1 (.minusDays now 14) row2 (.minusDays now 2) row3 (.plusDays now 25) row4 (.plusDays now 6) table (get-test-table [row1 row2 row3 row4] base-type)] (data/dataset (tx/dataset-definition db table) (testing "past12days" (is (= [[(->iso-str row2)]] (ctd/rows-without-index (qp/process-query (get-mbql "past12days" db)))))) (testing "next12days" (is (= [[(->iso-str row4)]] (ctd/rows-without-index (qp/process-query (get-mbql "next12days" db))))))))) (testing "past/next months/quarters" (let [db (format "mb_vars_m_q_%s" (u/lower-case-en base-type)) now (local-date-now) row1 (.minusMonths now 14) row2 (.minusMonths now 4) row3 (.plusMonths now 25) row4 (.plusMonths now 6) table (get-test-table [row1 row2 row3 row4] base-type)] (data/dataset (tx/dataset-definition db table) (testing "past12months" (is (= [[(->iso-str row2)]] (ctd/rows-without-index (qp/process-query (get-mbql "past12months" db)))))) (testing "next12months" (is (= [[(->iso-str row4)]] (ctd/rows-without-index (qp/process-query (get-mbql "next12months" db)))))) (testing "past3quarters" (is (= [[(->iso-str row2)]] (ctd/rows-without-index (qp/process-query (get-mbql "past3quarters" db)))))) (testing "next3quarters" (is (= [[(->iso-str row4)]] (ctd/rows-without-index (qp/process-query (get-mbql "next3quarters" db))))))))) (testing "past/next years" (let [db (format "mb_vars_y_%s" (u/lower-case-en base-type)) now (local-date-now) row1 (.minusYears now 14) row2 (.minusYears now 4) row3 (.plusYears now 25) row4 (.plusYears now 6) table (get-test-table [row1 row2 row3 row4] base-type)] (data/dataset (tx/dataset-definition db table) (testing "past12years" (is (= [[(->iso-str row2)]] (ctd/rows-without-index (qp/process-query (get-mbql "past12years" db)))))) (testing "next12years" (is (= [[(->iso-str row4)]] (ctd/rows-without-index (qp/process-query (get-mbql "next12years" db))))))))))))))) (deftest ^:parallel clickhouse-variables-field-filters-null-dates (mt/test-driver :clickhouse (mt/with-clock clock (letfn [(->input-ld [^LocalDate ld] [(t/format "yyyy-MM-dd" ld)]) (->input-ldt [^LocalDateTime ldt] [(t/format "yyyy-MM-dd HH:mm:ss" ldt)]) (->iso-str-ld [^LocalDate ld] (str (t/format "yyyy-MM-dd" ld) "T00:00:00Z")) (->iso-str-ldt [^LocalDateTime ldt] (t/format "yyyy-MM-dd'T'HH:mm:ss'Z'" ldt))] (let [db "mb_vars_null_dates" now-ld (local-date-now) now-ldt (local-date-time-now) table ["test_table" [{:field-name "d" :base-type {:native "Date"}} {:field-name "d32" :base-type {:native "Date32"}} {:field-name "dt" :base-type {:native "DateTime"}} {:field-name "dt64" :base-type {:native "DateTime64"}}] [;; row 1 [(->input-ld now-ld) nil (->input-ldt now-ldt) nil] ;; row 2 [nil (->input-ld now-ld) nil (->input-ldt now-ldt)]]] first-row [[(->iso-str-ld now-ld) nil (->iso-str-ldt now-ldt) nil]] second-row [[nil (->iso-str-ld now-ld) nil (->iso-str-ldt now-ldt)]]] (data/dataset (tx/dataset-definition db table) (letfn [(get-mbql* [field value] (let [uuid (str (java.util.UUID/randomUUID))] {:database (mt/id) :type "native" :native {:collection "test-table" :template-tags {:x {:id uuid :name (str field) :display-name (str field) :type "dimension" :dimension ["field" (mt/id :test-table field) nil] :required true}} :query (format "SELECT * FROM `%s`.`test_table` WHERE {{x}}" db)} :parameters [{:type "date/all-options" :value value :target ["dimension" ["template-tag" "x"]] :id uuid}]}))] (testing "first row (Date field match)" (is (= first-row (ctd/rows-without-index (qp/process-query (get-mbql* :d "2019-11-30")))))) (testing "first row (DateTime field match)" (is (= first-row (ctd/rows-without-index (qp/process-query (get-mbql* :dt "2019-11-30T23:00:00")))))) (testing "second row (Date32 field match)" (is (= second-row (ctd/rows-without-index (qp/process-query (get-mbql* :d32 "2019-11-30")))))) (testing "second row (DateTime64 field match)" (is (= second-row (ctd/rows-without-index (qp/process-query (get-mbql* :dt64 "2019-11-30T23:00:00"))))))))))))) ================================================ FILE: test/metabase/driver/clickhouse_temporal_bucketing_test.clj ================================================ (ns metabase.driver.clickhouse-temporal-bucketing-test #_{:clj-kondo/ignore [:unsorted-required-namespaces]} (:require [clojure.test :refer :all] [metabase.query-processor.test-util :as qp.test] [metabase.test :as mt] [metabase.test.data :as data] [metabase.test.data.clickhouse :as ctd])) (use-fixtures :once ctd/create-test-db!) ;; See temporal_bucketing table definition ;; Fields values are (both in server and column timezones): ;; start_of_year == '2022-01-01 00:00:00' ;; mid_of_year == '2022-06-20 06:32:54' ;; end_of_year == '2022-12-31 23:59:59' (deftest clickhouse-temporal-bucketing-server-tz (mt/test-driver :clickhouse (defn- start-of-year [unit] (qp.test/rows (ctd/do-with-test-db (fn [db] (data/with-db db (data/run-mbql-query temporal_bucketing_server_tz {:breakout [[:field %start_of_year {:temporal-unit unit}]]})))))) (defn- mid-year [unit] (qp.test/rows (ctd/do-with-test-db (fn [db] (data/with-db db (data/run-mbql-query temporal_bucketing_server_tz {:breakout [[:field %mid_of_year {:temporal-unit unit}]]})))))) (defn- end-of-year [unit] (qp.test/rows (ctd/do-with-test-db (fn [db] (data/with-db db (data/run-mbql-query temporal_bucketing_server_tz {:breakout [[:field %end_of_year {:temporal-unit unit}]]})))))) (testing "truncate to" (testing "minute" (is (= [["2022-06-20T06:32:00Z"]] (mid-year :minute)))) (testing "hour" (is (= [["2022-06-20T06:00:00Z"]] (mid-year :hour)))) (testing "day" (is (= [["2022-06-20T00:00:00Z"]] (mid-year :day)))) (testing "month" (is (= [["2022-06-01T00:00:00Z"]] (mid-year :month)))) (testing "quarter" (is (= [["2022-04-01T00:00:00Z"]] (mid-year :quarter)))) (testing "year" (is (= [["2022-01-01T00:00:00Z"]] (mid-year :year))))) (testing "extract" (testing "minute of hour" (is (= [[0]] (start-of-year :minute-of-hour))) (is (= [[32]] (mid-year :minute-of-hour))) (is (= [[59]] (end-of-year :minute-of-hour)))) (testing "hour of day" (is (= [[0]] (start-of-year :hour-of-day))) (is (= [[6]] (mid-year :hour-of-day))) (is (= [[23]] (end-of-year :hour-of-day)))) (testing "day of month" (is (= [[1]] (start-of-year :day-of-month))) (is (= [[20]] (mid-year :day-of-month))) (is (= [[31]] (end-of-year :day-of-month)))) (testing "day of year" (is (= [[1]] (start-of-year :day-of-year))) (is (= [[171]] (mid-year :day-of-year))) (is (= [[365]] (end-of-year :day-of-year)))) (testing "month of year" (is (= [[1]] (start-of-year :month-of-year))) (is (= [[6]] (mid-year :month-of-year))) (is (= [[12]] (end-of-year :month-of-year)))) (testing "quarter of year" (is (= [[1]] (start-of-year :quarter-of-year))) (is (= [[2]] (mid-year :quarter-of-year))) (is (= [[4]] (end-of-year :quarter-of-year))))))) (deftest clickhouse-temporal-bucketing-column-tz (mt/test-driver :clickhouse (defn- start-of-year [unit] (qp.test/rows (ctd/do-with-test-db (fn [db] (data/with-db db (data/run-mbql-query temporal_bucketing_column_tz {:breakout [[:field %start_of_year {:temporal-unit unit}]]})))))) (defn- mid-year [unit] (qp.test/rows (ctd/do-with-test-db (fn [db] (data/with-db db (data/run-mbql-query temporal_bucketing_column_tz {:breakout [[:field %mid_of_year {:temporal-unit unit}]]})))))) (defn- end-of-year [unit] (qp.test/rows (ctd/do-with-test-db (fn [db] (data/with-db db (data/run-mbql-query temporal_bucketing_column_tz {:breakout [[:field %end_of_year {:temporal-unit unit}]]})))))) (testing "truncate to" (testing "minute" (is (= [["2022-06-20T13:32:00Z"]] (mid-year :minute)))) (testing "hour" (is (= [["2022-06-20T13:00:00Z"]] (mid-year :hour)))) (testing "day" (is (= [["2022-06-20T07:00:00Z"]] (mid-year :day)))) (testing "month" (is (= [["2022-06-01T00:00:00Z"]] (mid-year :month)))) (testing "quarter" (is (= [["2022-04-01T00:00:00Z"]] (mid-year :quarter)))) (testing "year" (is (= [["2022-01-01T00:00:00Z"]] (mid-year :year))))) (testing "extract" (testing "minute of hour" (is (= [[0]] (start-of-year :minute-of-hour))) (is (= [[32]] (mid-year :minute-of-hour))) (is (= [[59]] (end-of-year :minute-of-hour)))) (testing "hour of day" (is (= [[0]] (start-of-year :hour-of-day))) (is (= [[6]] (mid-year :hour-of-day))) (is (= [[23]] (end-of-year :hour-of-day)))) (testing "day of month" (is (= [[1]] (start-of-year :day-of-month))) (is (= [[20]] (mid-year :day-of-month))) (is (= [[31]] (end-of-year :day-of-month)))) (testing "day of year" (is (= [[1]] (start-of-year :day-of-year))) (is (= [[171]] (mid-year :day-of-year))) (is (= [[365]] (end-of-year :day-of-year)))) (testing "month of year" (is (= [[1]] (start-of-year :month-of-year))) (is (= [[6]] (mid-year :month-of-year))) (is (= [[12]] (end-of-year :month-of-year)))) (testing "quarter of year" (is (= [[1]] (start-of-year :quarter-of-year))) (is (= [[2]] (mid-year :quarter-of-year))) (is (= [[4]] (end-of-year :quarter-of-year))))))) ================================================ FILE: test/metabase/driver/clickhouse_test.clj ================================================ (ns metabase.driver.clickhouse-test "Tests for specific behavior of the ClickHouse driver." #_{:clj-kondo/ignore [:unsorted-required-namespaces]} (:require [clojure.test :refer :all] [metabase.driver :as driver] [metabase.driver.clickhouse :as clickhouse] [metabase.driver.clickhouse-qp :as clickhouse-qp] [metabase.driver.sql-jdbc :as sql-jdbc] [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn] [metabase.query-processor.compile :as qp.compile] [metabase.test :as mt] [metabase.test.data :as data] [metabase.test.data.interface :as tx] [metabase.test.data.clickhouse :as ctd] [taoensso.nippy :as nippy] [toucan2.tools.with-temp :as t2.with-temp])) (set! *warn-on-reflection* true) (use-fixtures :once ctd/create-test-db!) ;; the mt/with-dynamic-redefs macro was renamed to mt/with-dynamic-fn-redefs for 0.53+ ;; as 0.52 is still tested by CI we will check which macro is defined and use that (defmacro with-dynamic-redefs [bindings & body] (if (resolve `mt/with-dynamic-redefs) `(mt/with-dynamic-redefs ~bindings ~@body) `(mt/with-dynamic-fn-redefs ~bindings ~@body))) (deftest ^:parallel clickhouse-version (mt/test-driver :clickhouse (t2.with-temp/with-temp [:model/Database db {:engine :clickhouse :details (tx/dbdef->connection-details :clickhouse :db {:database-name "default"})}] (let [version (driver/dbms-version :clickhouse db)] (is (number? (get-in version [:semantic-version :major]))) (is (number? (get-in version [:semantic-version :minor]))) (is (string? (get version :version))))))) (deftest ^:parallel clickhouse-server-timezone (mt/test-driver :clickhouse (is (= "UTC" (let [details (tx/dbdef->connection-details :clickhouse :db {:database-name "default"}) spec (sql-jdbc.conn/connection-details->spec :clickhouse details)] (driver/db-default-timezone :clickhouse spec)))))) (deftest ^:parallel clickhouse-connection-string (mt/with-dynamic-fn-redefs [ ;; This function's implementation requires the connection details to actually connect to the ;; database, which is orthogonal to the purpose of this test. clickhouse/cloud? (constantly false)] (testing "connection with no additional options" (is (= ctd/default-connection-params (sql-jdbc.conn/connection-details->spec :clickhouse {})))) (testing "custom connection with additional options" (is (= (merge ctd/default-connection-params {:subname "//myclickhouse:9999/foo?sessionTimeout=42" :user "bob" :password "qaz" :ssl true :custom_http_params "max_threads=42,allow_experimental_analyzer=0"}) (sql-jdbc.conn/connection-details->spec :clickhouse {:host "myclickhouse" :port 9999 :user "bob" :password "qaz" :dbname "foo" :additional-options "sessionTimeout=42" :ssl true :clickhouse-settings "max_threads=42,allow_experimental_analyzer=0"})))) (testing "nil dbname handling" (is (= ctd/default-connection-params (sql-jdbc.conn/connection-details->spec :clickhouse {:dbname nil})))) (testing "schema removal" (doall (for [host ["localhost" "http://localhost" "https://localhost"]] (testing (str "for host " host) (is (= ctd/default-connection-params (sql-jdbc.conn/connection-details->spec :clickhouse {:host host})))))) (doall (for [host ["myhost" "http://myhost" "https://myhost"]] (testing (str "for host " host) (is (= (merge ctd/default-connection-params {:subname "//myhost:8123/default"}) (sql-jdbc.conn/connection-details->spec :clickhouse {:host host})))))) (doall (for [host ["sub.example.com" "http://sub.example.com" "https://sub.example.com"]] (testing (str "for host " host " with some additional params") (is (= (merge ctd/default-connection-params {:subname "//sub.example.com:8443/mydb" :ssl true}) (sql-jdbc.conn/connection-details->spec :clickhouse {:host host :dbname "mydb" :port 8443 :ssl true}))))))))) (deftest ^:parallel clickhouse-connection-string-select-sequential-consistency (mt/with-dynamic-fn-redefs [ ;; This function's implementation requires the connection details to actually ;; connect to the database, which is orthogonal to the purpose of this test. clickhouse/cloud? (constantly true)] (testing "connection with no additional options" (is (= (assoc ctd/default-connection-params :select_sequential_consistency true) (sql-jdbc.conn/connection-details->spec :clickhouse {})))))) (deftest clickhouse-connection-fails-test (mt/test-driver :clickhouse (mt/with-temp [:model/Database db {:details (assoc (mt/db) :password "wrongpassword") :engine :clickhouse}] (testing "sense check that checking the cloud mode fails with a SQLException." ;; nil arg isn't tested here, as it will pick up the defaults, which is the same as the Docker instance credentials. (is (thrown? java.sql.SQLException (#'clickhouse/cloud? (:details db))))) (testing "`driver/database-supports? :uploads` does not throw even if the connection fails." (is (false? (driver/database-supports? :clickhouse :uploads db))) (is (false? (driver/database-supports? :clickhouse :uploads nil)))) (testing "`driver/database-supports? :connection-impersonation` does not throw even if the connection fails." (is (false? (driver/database-supports? :clickhouse :connection-impersonation db))) (is (false? (driver/database-supports? :clickhouse :connection-impersonation nil)))) (testing (str "`sql-jdbc.conn/connection-details->spec` does not throw even if the connection fails, " "and doesn't include the `select_sequential_consistency` parameter.") (is (nil? (:select_sequential_consistency (sql-jdbc.conn/connection-details->spec :clickhouse db)))) (is (nil? (:select_sequential_consistency (sql-jdbc.conn/connection-details->spec :clickhouse nil)))))))) (deftest ^:parallel clickhouse-tls (mt/test-driver :clickhouse (let [working-dir (System/getProperty "user.dir") cert-path (str working-dir "/modules/drivers/clickhouse/.docker/clickhouse/single_node_tls/certificates/ca.crt") additional-options (str "sslrootcert=" cert-path)] (testing "simple connection with a single database" (is (= "UTC" (driver/db-default-timezone :clickhouse (sql-jdbc.conn/connection-details->spec :clickhouse {:ssl true :host "server.clickhouseconnect.test" :port 8443 :additional-options additional-options}))))) (testing "connection with multiple databases" (is (= "UTC" (driver/db-default-timezone :clickhouse (sql-jdbc.conn/connection-details->spec :clickhouse {:ssl true :host "server.clickhouseconnect.test" :port 8443 :dbname "default system" :additional-options additional-options})))))))) (deftest ^:parallel clickhouse-nippy (mt/test-driver :clickhouse (testing "UnsignedByte" (let [value (com.clickhouse.data.value.UnsignedByte/valueOf "214")] (is (= value (nippy/thaw (nippy/freeze value)))))) (testing "UnsignedShort" (let [value (com.clickhouse.data.value.UnsignedShort/valueOf "62055")] (is (= value (nippy/thaw (nippy/freeze value)))))) (testing "UnsignedInteger" (let [value (com.clickhouse.data.value.UnsignedInteger/valueOf "4748364")] (is (= value (nippy/thaw (nippy/freeze value)))))) (testing "UnsignedLong" (let [value (com.clickhouse.data.value.UnsignedLong/valueOf "84467440737095")] (is (= value (nippy/thaw (nippy/freeze value)))))))) (deftest ^:parallel clickhouse-query-formatting (mt/test-driver :clickhouse (let [query (data/mbql-query venues {:fields [$id] :order-by [[:asc $id]] :limit 5}) {compiled :query} (qp.compile/compile-with-inline-parameters query) pretty (driver/prettify-native-form :clickhouse compiled)] (testing "compiled" (is (= "SELECT `test_data`.`venues`.`id` AS `id` FROM `test_data`.`venues` ORDER BY `test_data`.`venues`.`id` ASC LIMIT 5" compiled))) (testing "pretty" (is (= "SELECT\n `test_data`.`venues`.`id` AS `id`\nFROM\n `test_data`.`venues`\nORDER BY\n `test_data`.`venues`.`id` ASC\nLIMIT\n 5" pretty)))))) (deftest ^:parallel clickhouse-can-connect (mt/test-driver :clickhouse (doall (for [[username password] [["default" ""] ["user_with_password" "foo@bar!"]] database ["default" "Special@Characters~"]] (testing (format "User `%s` can connect to `%s`" username database) (let [details (merge {:user username :password password} (tx/dbdef->connection-details :clickhouse :db {:database-name database}))] (is (true? (driver/can-connect? :clickhouse details))))))))) (deftest clickhouse-qp-extract-datetime-timezone (mt/test-driver :clickhouse (is (= "utc" (#'clickhouse-qp/extract-datetime-timezone "datetime('utc')"))) (is (= "utc" (#'clickhouse-qp/extract-datetime-timezone "datetime64(3, 'utc')"))) (is (= "europe/amsterdam" (#'clickhouse-qp/extract-datetime-timezone "datetime('europe/amsterdam')"))) (is (= "europe/amsterdam" (#'clickhouse-qp/extract-datetime-timezone "datetime64(9, 'europe/amsterdam')"))) (is (= nil (#'clickhouse-qp/extract-datetime-timezone "datetime"))) (is (= nil (#'clickhouse-qp/extract-datetime-timezone "datetime64"))) (is (= nil (#'clickhouse-qp/extract-datetime-timezone "datetime64(3)"))))) (deftest ^:synchronized clickhouse-insert (mt/test-driver :clickhouse (t2.with-temp/with-temp [:model/Database db {:engine :clickhouse :details (tx/dbdef->connection-details :clickhouse :db {:database-name "default"})}] (let [table (keyword (format "insert_table_%s" (System/currentTimeMillis)))] (driver/create-table! :clickhouse (:id db) table {:id "Int64", :name "String"}) (try (driver/insert-into! :clickhouse (:id db) table [:id :name] [[42 "Bob"] [43 "Alice"]]) (is (= #{{:id 42, :name "Bob"} {:id 43, :name "Alice"}} (set (sql-jdbc/query :clickhouse db {:select [:*] :from [table]})))) (finally (driver/drop-table! :clickhouse (:id db) table))))))) ================================================ FILE: test/metabase/test/data/clickhouse.clj ================================================ (ns metabase.test.data.clickhouse "Code for creating / destroying a ClickHouse database from a `DatabaseDefinition`." (:require [clojure.java.io :as io] [clojure.java.jdbc :as jdbc] [clojure.string :as str] [clojure.test :refer :all] [java-time.api :as t] [metabase.db.query :as mdb.query] [metabase.driver :as driver] [metabase.driver.ddl.interface :as ddl.i] [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn] [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute] [metabase.driver.sql.util :as sql.u] [metabase.lib.schema.common :as lib.schema.common] [metabase.query-processor-test.alternative-date-test :as qp.alternative-date-test] [metabase.query-processor.test-util :as qp.test] [metabase.sync.core :as sync] [metabase.test.data.interface :as tx] [metabase.test.data.sql :as sql.tx] [metabase.test.data.sql-jdbc :as sql-jdbc.tx] [metabase.test.data.sql-jdbc.execute :as execute] [metabase.test.data.sql-jdbc.load-data :as load-data] [metabase.test.data.sql.ddl :as ddl] [metabase.util.log :as log] [metabase.util.malli :as mu] [toucan2.tools.with-temp :as t2.with-temp])) (sql-jdbc.tx/add-test-extensions! :clickhouse) (defmethod driver/database-supports? [:clickhouse :metabase.driver.sql-jdbc.sync.describe-table-test/describe-view-fields] [_driver _feature _db] true) (defmethod driver/database-supports? [:clickhouse :metabase.driver.sql-jdbc.sync.describe-table-test/describe-materialized-view-fields] [_driver _feature _db] false) (defmethod driver/database-supports? [:clickhouse :metabase.query-processor-test.parameters-test/get-parameter-count] [_driver _feature _db] false) (defmethod qp.alternative-date-test/iso-8601-text-fields-expected-rows :clickhouse [_driver] [[1 "foo" (t/offset-date-time "2004-10-19T10:23:54Z") #t "2004-10-19" (t/offset-date-time "1970-01-01T10:23:54Z")] [2 "bar" (t/offset-date-time "2008-10-19T10:23:54Z") #t "2008-10-19" (t/offset-date-time "1970-01-01T10:23:54Z")] [3 "baz" (t/offset-date-time "2012-10-19T10:23:54Z") #t "2012-10-19" (t/offset-date-time "1970-01-01T10:23:54Z")]]) (def default-connection-params {:classname "com.clickhouse.jdbc.ClickHouseDriver" :subprotocol "clickhouse" :subname "//localhost:8123/default" :user "default" :password "" :ssl false :use_server_time_zone_for_dates true :product_name "metabase/1.53.4" :jdbc_ignore_unsupported_values "true" :jdbc_schema_term "schema", :max_open_connections 100 :remember_last_set_roles true :http_connection_provider "HTTP_URL_CONNECTION" :custom_http_params ""}) (defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/Boolean] [_ _] "Boolean") (defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/BigInteger] [_ _] "Int64") (defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/Char] [_ _] "String") (defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/Date] [_ _] "Date") (defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/DateTime] [_ _] "DateTime64(3, 'GMT0')") (defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/DateTimeWithLocalTZ] [_ _] "DateTime64(3, 'UTC')") (defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/Float] [_ _] "Float64") (defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/Integer] [_ _] "Int32") (defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/IPAddress] [_ _] "IPv4") (defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/Text] [_ _] "String") (defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/UUID] [_ _] "UUID") (defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/Time] [_ _] "Time") (defmethod tx/sorts-nil-first? :clickhouse [_ _] false) (defmethod tx/dbdef->connection-details :clickhouse [_ context {:keys [database-name]}] (merge {:host (tx/db-test-env-var-or-throw :clickhouse :host "localhost") :port (tx/db-test-env-var-or-throw :clickhouse :port 8123) :timezone :America/Los_Angeles} (when-let [user (tx/db-test-env-var :clickhouse :user)] {:user user}) (when-let [password (tx/db-test-env-var :clickhouse :password)] {:password password}) (when (= context :db) {:db database-name}))) (defmethod sql.tx/qualified-name-components :clickhouse ([_ db-name] [db-name]) ([_ db-name table-name] [db-name table-name]) ([_ db-name table-name field-name] [db-name table-name field-name])) (defmethod tx/create-db! :clickhouse [driver {:keys [database-name], :as db-def} & options] (let [database-name (ddl.i/format-name driver database-name)] (log/infof "Creating ClickHouse database %s" (pr-str database-name)) ;; call the default impl for SQL JDBC drivers (apply (get-method tx/create-db! :sql-jdbc/test-extensions) driver db-def options))) (defmethod ddl/insert-rows-dml-statements :clickhouse [driver table-identifier rows] (binding [driver/*compile-with-inline-parameters* true] ((get-method ddl/insert-rows-dml-statements :sql-jdbc/test-extensions) driver table-identifier rows))) (mu/defmethod load-data/do-insert! :clickhouse [driver :- :keyword ^java.sql.Connection conn :- (lib.schema.common/instance-of-class java.sql.Connection) table-identifier rows] (let [statements (ddl/insert-rows-dml-statements driver table-identifier rows)] (doseq [sql-args statements :let [sql-args (if (string? sql-args) [sql-args] sql-args)]] (assert (string? (first sql-args)) (format "Bad sql-args: %s" (pr-str sql-args))) (log/tracef "[insert] %s" (pr-str sql-args)) (try (jdbc/execute! {:connection conn :transaction? false} sql-args {:set-parameters (fn [stmt params] (sql-jdbc.execute/set-parameters! driver stmt params))}) (catch Throwable e (throw (ex-info (format "INSERT FAILED: %s" (ex-message e)) {:driver driver :sql-args (into [(str/split-lines (mdb.query/format-sql (first sql-args)))] (rest sql-args))} e))))))) (defn- quote-name [name] (sql.u/quote-name :clickhouse :field (ddl.i/format-name :clickhouse name))) (def ^:private non-nullable-types ["Array" "Map" "Tuple"]) (defn- disallowed-as-nullable? [ch-type] (boolean (some #(str/starts-with? ch-type %) non-nullable-types))) (defn- field->clickhouse-column [field] (let [{:keys [field-name base-type pk?]} field ch-type (if (map? base-type) (:native base-type) (sql.tx/field-base-type->sql-type :clickhouse base-type)) col-name (quote-name field-name) ch-col (cond (or pk? (disallowed-as-nullable? ch-type)) (format "%s %s" col-name ch-type) (= ch-type "Time") (format "%s Nullable(DateTime64) COMMENT 'time'" col-name) ; _ :else (format "%s Nullable(%s)" col-name ch-type))] ch-col)) (defn- ->comma-separated-str [coll] (->> coll (interpose ", ") (apply str))) (defmethod sql.tx/create-table-sql :clickhouse [_ {:keys [database-name]} {:keys [table-name field-definitions]}] (let [table-name (sql.tx/qualify-and-quote :clickhouse database-name table-name) pk-fields (filter (fn [{:keys [pk?]}] pk?) field-definitions) pk-field-names (map #(quote-name (:field-name %)) pk-fields) fields (->> field-definitions (map field->clickhouse-column) (->comma-separated-str)) order-by (->comma-separated-str pk-field-names)] (format "CREATE TABLE %s (%s) ENGINE = MergeTree ORDER BY (%s) SETTINGS allow_nullable_key=1" table-name fields order-by))) (defmethod execute/execute-sql! :clickhouse [& args] (apply execute/sequentially-execute-sql! args)) (defmethod load-data/row-xform :clickhouse [_driver _dbdef tabledef] (load-data/maybe-add-ids-xform tabledef)) (defmethod sql.tx/pk-sql-type :clickhouse [_] "Int32") (defmethod sql.tx/add-fk-sql :clickhouse [& _] nil) (defmethod sql.tx/session-schema :clickhouse [_] "default") (defn rows-without-index "Remove the Metabase index which is the first column in the result set" [query-result] (map #(drop 1 %) (qp.test/rows query-result))) (def ^:private test-db-initialized? (atom false)) (defn create-test-db! "Create a ClickHouse database called `metabase_test` and initialize some test data" [f] (when (not @test-db-initialized?) (let [details (tx/dbdef->connection-details :clickhouse :db {:database-name "metabase_test"})] ;; (println "### Executing create-test-db! with details:" details) (jdbc/with-db-connection [spec (sql-jdbc.conn/connection-details->spec :clickhouse (merge {:engine :clickhouse} details))] (let [raw-statements (slurp (io/resource "metabase/test/data/clickhouse_datasets.sql")) statements (as-> raw-statements s (str/split s #";") (map str/trim s) (filter seq s))] ;; (println "## Executing statements " statements) (jdbc/db-do-commands spec false statements) (reset! test-db-initialized? true))) ;; (println "### Done with executing create-test-db! with details:" details) )) (f)) #_{:clj-kondo/ignore [:warn-on-reflection]} (defn exec-statements ([statements details-map] (exec-statements statements details-map nil)) ([statements details-map clickhouse-settings] (sql-jdbc.execute/do-with-connection-with-options :clickhouse (sql-jdbc.conn/connection-details->spec :clickhouse (merge {:engine :clickhouse} details-map)) {:write? true} (fn [^java.sql.Connection conn] (doseq [statement statements] ;; (println "Executing:" statement) (let [^com.clickhouse.jdbc.ConnectionImpl clickhouse-conn (.unwrap conn com.clickhouse.jdbc.ConnectionImpl) query-settings (new com.clickhouse.client.api.query.QuerySettings)] (with-open [jdbcStmt (.createStatement conn)] (when clickhouse-settings (doseq [[k v] clickhouse-settings] (.setOption query-settings k v))) (.setDefaultQuerySettings clickhouse-conn query-settings) (.execute jdbcStmt statement)))))))) (defn do-with-test-db "Execute a test function using the test dataset" [f] (t2.with-temp/with-temp [:model/Database database {:engine :clickhouse :details (tx/dbdef->connection-details :clickhouse :db {:database-name "metabase_test"})}] (sync/sync-db-metadata! database) (f database))) (defmethod tx/dataset-already-loaded? :clickhouse [driver dbdef] (let [tabledef (first (:table-definitions dbdef)) db-name (ddl.i/format-name :clickhouse (:database-name dbdef)) table-name (ddl.i/format-name :clickhouse (:table-name tabledef)) details (tx/dbdef->connection-details :clickhouse :db {:database-name db-name})] (sql-jdbc.execute/do-with-connection-with-options driver (sql-jdbc.conn/connection-details->spec driver details) {:write? false} (fn [^java.sql.Connection conn] (with-open [rset (.getTables (.getMetaData conn) #_catalog nil #_schema-pattern db-name #_table-pattern table-name #_types (into-array String ["TABLE"]))] ;; if the ResultSet returns anything we know the table is already loaded. (.next rset)))))) ================================================ FILE: test/metabase/test/data/clickhouse_datasets.sql ================================================ DROP DATABASE IF EXISTS `metabase_test`; CREATE DATABASE `metabase_test`; CREATE TABLE `metabase_test`.`metabase_test_lowercases` ( id UInt8, mystring Nullable(String) ) ENGINE = Memory; INSERT INTO `metabase_test`.`metabase_test_lowercases` VALUES (1, 'Я_1'), (2, 'R'), (3, 'Я_2'), (4, 'Я'), (5, 'я'), (6, NULL); CREATE TABLE `metabase_test`.`enums_test` ( enum1 Enum8('foo' = 0, 'bar' = 1, 'foo bar' = 2), enum2 Enum16('click' = 0, 'house' = 1), enum3 Enum8('qaz' = 42, 'qux' = 23) ) ENGINE = Memory; INSERT INTO `metabase_test`.`enums_test` (enum1, enum2, enum3) VALUES ('foo', 'house', 'qaz'), ('foo bar', 'click', 'qux'), ('bar', 'house', 'qaz'); CREATE TABLE `metabase_test`.`ipaddress_test` ( ipvfour Nullable(IPv4), ipvsix Nullable(IPv6) ) Engine = Memory; INSERT INTO `metabase_test`.`ipaddress_test` (ipvfour, ipvsix) VALUES (toIPv4('127.0.0.1'), toIPv6('0:0:0:0:0:0:0:1')), (toIPv4('0.0.0.0'), toIPv6('2001:438:ffff:0:0:0:407d:1bc1')), (null, null); CREATE TABLE `metabase_test`.`boolean_test` ( ID Int32, b1 Bool, b2 Nullable(Bool) ) ENGINE = Memory; INSERT INTO `metabase_test`.`boolean_test` (ID, b1, b2) VALUES (1, true, true), (2, false, true), (3, true, false); CREATE TABLE `metabase_test`.`maps_test` ( m Map(String, UInt64) ) ENGINE = Memory; INSERT INTO `metabase_test`.`maps_test` VALUES ({'key1':1,'key2':10}), ({'key1':2,'key2':20}), ({'key1':3,'key2':30}); CREATE TABLE `metabase_test`.`array_of_tuples_test` ( t Array(Tuple(String, UInt32)) ) Engine = Memory; INSERT INTO `metabase_test`.`array_of_tuples_test` (t) VALUES ([('foobar', 1234), ('qaz', 0)]), ([]); -- Used for testing that AggregateFunction columns are excluded, -- while SimpleAggregateFunction columns are preserved CREATE TABLE `metabase_test`.`aggregate_functions_filter_test` ( idx UInt8, a AggregateFunction(uniq, String), lowest_value SimpleAggregateFunction(min, UInt8), count SimpleAggregateFunction(sum, Int64) ) ENGINE Memory; INSERT INTO `metabase_test`.`aggregate_functions_filter_test` (idx, lowest_value, count) VALUES (42, 144, 255255); -- Materialized views (testing .inner tables exclusion) CREATE TABLE `metabase_test`.`wikistat` ( `date` Date, `project` LowCardinality(String), `hits` UInt32 ) ENGINE = Memory; CREATE MATERIALIZED VIEW `metabase_test`.`wikistat_mv` ENGINE =Memory AS SELECT date, project, sum(hits) AS hits FROM `metabase_test`.`wikistat` GROUP BY date, project; INSERT INTO `metabase_test`.`wikistat` VALUES (now(), 'foo', 10), (now(), 'bar', 10), (now(), 'bar', 20); -- Used in sum-where tests CREATE TABLE `metabase_test`.`sum_if_test_int` ( id Int64, int_value Int64, discriminator String ) ENGINE = Memory; INSERT INTO `metabase_test`.`sum_if_test_int` VALUES (1, 1, 'foo'), (2, 1, 'foo'), (3, 3, 'bar'), (4, 5, 'bar'); CREATE TABLE `metabase_test`.`sum_if_test_float` ( id Int64, float_value Float64, discriminator String ) ENGINE = Memory; INSERT INTO `metabase_test`.`sum_if_test_float` VALUES (1, 1.1, 'foo'), (2, 1.44, 'foo'), (3, 3.5, 'bar'), (4, 5.77, 'bar'); -- Temporal bucketing tests CREATE TABLE `metabase_test`.`temporal_bucketing_server_tz` ( start_of_year DateTime, mid_of_year DateTime, end_of_year DateTime ) ENGINE = Memory; INSERT INTO `metabase_test`.`temporal_bucketing_server_tz` VALUES ('2022-01-01 00:00:00', '2022-06-20 06:32:54', '2022-12-31 23:59:59'); CREATE TABLE `metabase_test`.`temporal_bucketing_column_tz` ( start_of_year DateTime('America/Los_Angeles'), mid_of_year DateTime('America/Los_Angeles'), end_of_year DateTime('America/Los_Angeles') ) ENGINE = Memory; INSERT INTO `metabase_test`.`temporal_bucketing_column_tz` VALUES (toDateTime('2022-01-01 00:00:00', 'America/Los_Angeles'), toDateTime('2022-06-20 06:32:54', 'America/Los_Angeles'), toDateTime('2022-12-31 23:59:59', 'America/Los_Angeles')); CREATE TABLE `metabase_test`.`datetime_diff_nullable` ( idx Int32, dt64 Nullable(DateTime64(3, 'UTC')), dt Nullable(DateTime('UTC')), d Nullable(Date) ) ENGINE Memory; INSERT INTO `metabase_test`.`datetime_diff_nullable` VALUES (42, '2022-01-01 00:00:00.000', '2022-06-20 06:32:54', '2022-07-22'), (43, '2022-01-01 00:00:00.000', NULL, NULL), (44, NULL, '2022-06-20 06:32:54', '2022-07-22'), (45, NULL, NULL, NULL); DROP DATABASE IF EXISTS `metabase_db_scan_test`; CREATE DATABASE `metabase_db_scan_test`; CREATE TABLE `metabase_db_scan_test`.`table1` (i Int32) ENGINE = Memory; CREATE TABLE `metabase_db_scan_test`.`table2` (i Int64) ENGINE = Memory; -- Base type matching tests CREATE TABLE `metabase_test`.`enums_base_types` ( c1 Nullable(Enum8('America/New_York')), c2 Enum8('BASE TABLE' = 1, 'VIEW' = 2, 'FOREIGN TABLE' = 3, 'LOCAL TEMPORARY' = 4, 'SYSTEM VIEW' = 5), c3 Enum8('NO', 'YES'), c4 Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2), c5 Nullable(Enum8('GLOBAL' = 0, 'DATABASE' = 1, 'TABLE' = 2)), c6 Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2)) ) ENGINE Memory; CREATE TABLE `metabase_test`.`date_base_types` ( c1 Date, c2 Date32, c3 Nullable(Date), c4 Nullable(Date32) ) ENGINE Memory; CREATE TABLE `metabase_test`.`datetime_base_types` ( c1 Nullable(DateTime('America/New_York')), c2 DateTime('America/New_York'), c3 DateTime, c4 DateTime64(3), c5 DateTime64(9, 'America/New_York'), c6 Nullable(DateTime64(6, 'America/New_York')), c7 Nullable(DateTime64(0)), c8 Nullable(DateTime) ) ENGINE Memory; CREATE TABLE `metabase_test`.`integer_base_types` ( c1 UInt8, c2 UInt16, c3 UInt32, c4 UInt64, c5 UInt128, c6 UInt256, c7 Int8, c8 Int16, c9 Int32, c10 Int64, c11 Int128, c12 Int256, c13 Nullable(Int32) ) ENGINE Memory; CREATE TABLE `metabase_test`.`numeric_base_types` ( c1 Float32, c2 Float64, c3 Decimal(4, 2), c4 Decimal32(7), c5 Decimal64(12), c6 Decimal128(24), c7 Decimal256(42), c8 Nullable(Float32), c9 Nullable(Decimal(4, 2)), c10 Nullable(Decimal256(42)) ) ENGINE Memory; CREATE TABLE `metabase_test`.`string_base_types` ( c1 String, c2 LowCardinality(String), c3 FixedString(32), c4 Nullable(String), c5 LowCardinality(FixedString(4)) ) ENGINE Memory; CREATE TABLE `metabase_test`.`misc_base_types` ( c1 Boolean, c2 UUID, c3 IPv4, c4 IPv6, c5 Map(Int32, String), c6 Nullable(Boolean), c7 Nullable(UUID), c8 Nullable(IPv4), c9 Nullable(IPv6), c10 Tuple(String, Int32) ) ENGINE Memory; CREATE TABLE `metabase_test`.`array_base_types` ( c1 Array(String), c2 Array(Nullable(Int32)), c3 Array(Array(LowCardinality(FixedString(32)))), c4 Array(Array(Array(String))) ) ENGINE Memory; CREATE TABLE `metabase_test`.`low_cardinality_nullable_base_types` ( c1 LowCardinality(Nullable(String)), c2 LowCardinality(Nullable(FixedString(16))) ) ENGINE Memory; -- can-connect tests (odd database names) DROP DATABASE IF EXISTS `Special@Characters~`; CREATE DATABASE `Special@Characters~`; -- arrays inner types test CREATE TABLE `metabase_test`.`arrays_inner_types` ( `arr_str` Array(String), `arr_nstr` Array(Nullable(String)), `arr_dec` Array(Decimal(18, 4)), `arr_ndec` Array(Nullable(Decimal(18, 4))) ) ENGINE Memory; INSERT INTO `metabase_test`.`arrays_inner_types` VALUES ( ['a', 'b', 'c'], [NULL, 'd', 'e'], [1, 2, 3], [4, NULL, 5] ); CREATE TABLE `metabase_test`.`unsigned_int_types` ( `u8` UInt8, `u16` UInt16, `u32` UInt32, `u64` UInt64 ) ENGINE Memory; INSERT INTO `metabase_test`.`unsigned_int_types` VALUES (255, 65535, 4294967295, 18446744073709551615); CREATE TABLE `metabase_test`.`fixed_strings` ( `f1` FixedString(4), `f2` LowCardinality(FixedString(4)), `f3` Nullable(FixedString(4)), `f4` LowCardinality(Nullable(FixedString(4))) ) ENGINE Memory; INSERT INTO `metabase_test`.`fixed_strings` VALUES ('val1', 'val2', 'val3', 'val4');