[ { "path": ".build/Dockerfile", "content": "ARG METABASE_VERSION\nFROM metabase/metabase:$METABASE_VERSION\nCOPY ./clickhouse.metabase-driver.jar /plugins/clickhouse.metabase-driver.jar\nRUN chmod 744 /plugins/clickhouse.metabase-driver.jar\n" }, { "path": ".docker/clickhouse/cluster/server1_config.xml", "content": "\n\n\n 8123\n 9009\n clickhouse1\n\n users.xml\n default\n default\n\n 5368709120\n\n /var/lib/clickhouse/\n /var/lib/clickhouse/tmp/\n /var/lib/clickhouse/user_files/\n /var/lib/clickhouse/access/\n 3\n\n \n debug\n /var/log/clickhouse-server/clickhouse-server.log\n /var/log/clickhouse-server/clickhouse-server.err.log\n 1000M\n 10\n 1\n \n\n \n \n \n \n clickhouse1\n 9000\n \n \n clickhouse2\n 9000\n \n \n \n \n\n \n 9181\n 1\n /var/lib/clickhouse/coordination/log\n /var/lib/clickhouse/coordination/snapshots\n\n \n 10000\n 30000\n trace\n 10000\n \n\n \n \n 1\n clickhouse1\n 9000\n \n \n 2\n clickhouse2\n 9000\n \n \n \n\n \n \n clickhouse1\n 9181\n \n \n clickhouse2\n 9181\n \n \n\n \n /clickhouse/test_cluster/task_queue/ddl\n \n\n \n system\n query_log
\n toYYYYMM(event_date)\n 1000\n \n\n \n /var/lib/clickhouse/format_schemas/\n \n \n users.xml\n \n \n\n" }, { "path": ".docker/clickhouse/cluster/server1_macros.xml", "content": "\n \n test_cluster\n clickhouse1\n 1\n \n\n" }, { "path": ".docker/clickhouse/cluster/server2_config.xml", "content": "\n\n\n 8123\n 9009\n clickhouse2\n\n users.xml\n default\n default\n\n 5368709120\n\n /var/lib/clickhouse/\n /var/lib/clickhouse/tmp/\n /var/lib/clickhouse/user_files/\n /var/lib/clickhouse/access/\n 3\n\n \n debug\n /var/log/clickhouse-server/clickhouse-server.log\n /var/log/clickhouse-server/clickhouse-server.err.log\n 1000M\n 10\n 1\n \n\n \n \n \n \n clickhouse1\n 9000\n \n \n clickhouse2\n 9000\n \n \n \n \n\n \n 9181\n 2\n /var/lib/clickhouse/coordination/log\n /var/lib/clickhouse/coordination/snapshots\n\n \n 10000\n 30000\n trace\n 10000\n \n\n \n \n 1\n clickhouse1\n 9000\n \n \n 2\n clickhouse2\n 9000\n \n \n \n\n \n \n clickhouse1\n 9181\n \n \n clickhouse2\n 9181\n \n \n\n \n /clickhouse/test_cluster/task_queue/ddl\n \n\n \n system\n query_log
\n toYYYYMM(event_date)\n 1000\n \n\n \n /var/lib/clickhouse/format_schemas/\n \n \n users.xml\n \n \n\n" }, { "path": ".docker/clickhouse/cluster/server2_macros.xml", "content": "\n \n test_cluster\n clickhouse2\n 1\n \n\n" }, { "path": ".docker/clickhouse/single_node/config.xml", "content": "\n\n\n 8123\n 9000\n\n users.xml\n default\n default\n\n 5368709120\n\n /var/lib/clickhouse/\n /var/lib/clickhouse/tmp/\n /var/lib/clickhouse/user_files/\n /var/lib/clickhouse/access/\n UTC\n\n \n debug\n /var/log/clickhouse-server/clickhouse-server.log\n /var/log/clickhouse-server/clickhouse-server.err.log\n 1000M\n 10\n 1\n \n\n \n system\n query_log
\n toYYYYMM(event_date)\n 1000\n \n\n \n /var/lib/clickhouse/format_schemas/\n \n \n users.xml\n \n \n\n" }, { "path": ".docker/clickhouse/single_node/users.xml", "content": "\n\n\n \n \n random\n 1\n \n \n\n \n \n \n \n ::/0\n \n default\n default\n 1\n \n \n foo@bar!\n \n ::/0\n \n default\n default\n 1\n \n \n\n \n \n \n 3600\n 0\n 0\n 0\n 0\n 0\n \n \n \n\n" }, { "path": ".docker/clickhouse/single_node_tls/Dockerfile", "content": "FROM clickhouse/clickhouse-server:25.2-alpine\nCOPY .docker/clickhouse/single_node_tls/certificates /etc/clickhouse-server/certs\nRUN chown clickhouse:clickhouse -R /etc/clickhouse-server/certs \\\n && chmod 600 /etc/clickhouse-server/certs/* \\\n && chmod 755 /etc/clickhouse-server/certs\n" }, { "path": ".docker/clickhouse/single_node_tls/certificates/ca.crt", "content": "-----BEGIN CERTIFICATE-----\nMIICTTCCAdKgAwIBAgIUaqbLNiwUtbV5VuolTMGXOO+21vEwCgYIKoZIzj0EAwQw\nXTELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMSAwHgYDVQQKDBdDbGlja0hvdXNl\nIENvbm5lY3QgVGVzdDEfMB0GA1UEAwwWY2xpY2tob3VzZWNvbm5lY3QudGVzdDAe\nFw0yMjA1MTkxODIxMzFaFw00MjA1MTQxODIxMzFaMF0xCzAJBgNVBAYTAlVTMQsw\nCQYDVQQIDAJDQTEgMB4GA1UECgwXQ2xpY2tIb3VzZSBDb25uZWN0IFRlc3QxHzAd\nBgNVBAMMFmNsaWNraG91c2Vjb25uZWN0LnRlc3QwdjAQBgcqhkjOPQIBBgUrgQQA\nIgNiAATTKvPxkWILniWZ9EmcftQRqhH7fpVhQm1hvtZW1cpTozV0z6tdopnS5p/W\nl+Kti2k/kZx1rsN1ZrRYKJN8ANruJJ6vaDOjbf89cmViZ/dbOi49T8brTzdHeuGI\nE2TyP+WjUzBRMB0GA1UdDgQWBBThZgdf9aToyK2TeSQ+suyjNUuifDAfBgNVHSME\nGDAWgBThZgdf9aToyK2TeSQ+suyjNUuifDAPBgNVHRMBAf8EBTADAQH/MAoGCCqG\nSM49BAMEA2kAMGYCMQDWQUTb39xLLds0WobJmNQbIkEwZyss0XNQkn6qI8rz73NL\n6L5/6wNzetKhBf3WBCYCMQC+evVR3Td+WLfbKQDXrCbSkogW6++I/9l55wakMz9G\nP+0she/nvFuUKnB+VRcaBqM=\n-----END CERTIFICATE-----\n" }, { "path": ".docker/clickhouse/single_node_tls/certificates/ca.key", "content": "-----BEGIN EC PARAMETERS-----\nBgUrgQQAIg==\n-----END EC PARAMETERS-----\n-----BEGIN EC PRIVATE KEY-----\nMIGkAgEBBDA26CfROCd9PBA7jUKmImXeYSuExnZcdNeX6AR0Aaj1wgZ/iH3hoQYH\nbPi7JnRTo+6gBwYFK4EEACKhZANiAATTKvPxkWILniWZ9EmcftQRqhH7fpVhQm1h\nvtZW1cpTozV0z6tdopnS5p/Wl+Kti2k/kZx1rsN1ZrRYKJN8ANruJJ6vaDOjbf89\ncmViZ/dbOi49T8brTzdHeuGIE2TyP+U=\n-----END EC PRIVATE KEY-----\n" }, { "path": ".docker/clickhouse/single_node_tls/certificates/client.crt", "content": "-----BEGIN CERTIFICATE-----\nMIIB+TCCAX8CFEc86vC0vsMjLzQzxazHeHjQblL2MAoGCCqGSM49BAMEMF0xCzAJ\nBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEgMB4GA1UECgwXQ2xpY2tIb3VzZSBDb25u\nZWN0IFRlc3QxHzAdBgNVBAMMFmNsaWNraG91c2Vjb25uZWN0LnRlc3QwHhcNMjIw\nNTE5MjEwNTA2WhcNNDIwNTEzMjEwNTA2WjBkMQswCQYDVQQGEwJVUzELMAkGA1UE\nCAwCQ0ExIDAeBgNVBAoMF0NsaWNrSG91c2UgQ29ubmVjdCBUZXN0MSYwJAYDVQQD\nDB1jbGllbnQuY2xpY2tob3VzZWNvbm5lY3QudGVzdDB2MBAGByqGSM49AgEGBSuB\nBAAiA2IABBrSSv+9xHsp8Bge3wdoO+3VdDM4DDrocE0Gm+EW65MN6/6oDmbyKOB1\nJbTY0aq3lIN9PtUibCrGDqcVqtQnihnvTIDLqK0Xlxvv6Jc0t6DvXYaKhg6jIimt\nB7NEvysGVzAKBggqhkjOPQQDBANoADBlAjBblevbpaRlekX7fH16KnYttGoIqDBI\n45LlBJ2sEe5qSKCBoLdN89Tk8WD4lG7PhlkCMQDdFd8OKMPaZiUWIdHI6AeDWwXD\nbJi0LwDxXgyBVCGLZ2vTbOVxnr2Qp+9BjFURU8c=\n-----END CERTIFICATE-----\n" }, { "path": ".docker/clickhouse/single_node_tls/certificates/client.key", "content": "-----BEGIN EC PARAMETERS-----\nBgUrgQQAIg==\n-----END EC PARAMETERS-----\n-----BEGIN EC PRIVATE KEY-----\nMIGkAgEBBDCsmtNREbSJvGTTIQmx019frOllv9m2EqOWesvqu52rH6tIxdw5TE4A\nSZICvNKYzH2gBwYFK4EEACKhZANiAAQa0kr/vcR7KfAYHt8HaDvt1XQzOAw66HBN\nBpvhFuuTDev+qA5m8ijgdSW02NGqt5SDfT7VImwqxg6nFarUJ4oZ70yAy6itF5cb\n7+iXNLeg712GioYOoyIprQezRL8rBlc=\n-----END EC PRIVATE KEY-----\n" }, { "path": ".docker/clickhouse/single_node_tls/certificates/server.crt", "content": "-----BEGIN CERTIFICATE-----\nMIIDPTCCAsOgAwIBAgIURzzq8LS+wyMvNDPFrMd4eNBuUvUwCgYIKoZIzj0EAwQw\nXTELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMSAwHgYDVQQKDBdDbGlja0hvdXNl\nIENvbm5lY3QgVGVzdDEfMB0GA1UEAwwWY2xpY2tob3VzZWNvbm5lY3QudGVzdDAe\nFw0yMjA1MTkyMDU3MjRaFw00MjA1MTMyMDU3MjRaMGQxCzAJBgNVBAYTAlVTMQsw\nCQYDVQQIDAJDQTEgMB4GA1UECgwXQ2xpY2tIb3VzZSBDb25uZWN0IFRlc3QxJjAk\nBgNVBAMMHXNlcnZlci5jbGlja2hvdXNlY29ubmVjdC50ZXN0MHYwEAYHKoZIzj0C\nAQYFK4EEACIDYgAECsvHRYxPr+kJ/A7DDajEu8PhdO+WGxzJs7k9SdypPWSxOaCD\nME2tWq0t0Giy63JYNhsn+CJglNIXhtfS5nHS7NV5SfBABUVtZS2/MFk8CwFCz+Rc\nZ4db2gt937AgjfxCo4IBOzCCATcwCQYDVR0TBAIwADARBglghkgBhvhCAQEEBAMC\nBkAwOQYJYIZIAYb4QgENBCwWKkNsaWNrSG91c2UgQ29ubmVjdCBUZXN0IFNlcnZl\nciBDZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQUZDd2tpXw4FMDFcY38eXCb+tmukAwgZoG\nA1UdIwSBkjCBj4AU4WYHX/Wk6Mitk3kkPrLsozVLonyhYaRfMF0xCzAJBgNVBAYT\nAlVTMQswCQYDVQQIDAJDQTEgMB4GA1UECgwXQ2xpY2tIb3VzZSBDb25uZWN0IFRl\nc3QxHzAdBgNVBAMMFmNsaWNraG91c2Vjb25uZWN0LnRlc3SCFGqmyzYsFLW1eVbq\nJUzBlzjvttbxMAsGA1UdDwQEAwIF4DATBgNVHSUEDDAKBggrBgEFBQcDATAKBggq\nhkjOPQQDBANoADBlAjBc3W/8qr04xmUiDOHSEoug89cK8YxtRiKdCjiR3Lao1h5a\nJ5Xc0JhVLaDUFb+blkoCMQCM7rKbO3itBKaweeJijX/veBcISYFulryWeANiltxo\nDFDHrC54rGXt4eOMouTlPbw=\n-----END CERTIFICATE-----\n" }, { "path": ".docker/clickhouse/single_node_tls/certificates/server.key", "content": "-----BEGIN EC PARAMETERS-----\nBgUrgQQAIg==\n-----END EC PARAMETERS-----\n-----BEGIN EC PRIVATE KEY-----\nMIGkAgEBBDDyhVuBtGuyKEDr2HciKi4yS2T0WloMeUG2kgQRKim7Mih7977q7RbI\nt6sGwlsBxKGgBwYFK4EEACKhZANiAAQKy8dFjE+v6Qn8DsMNqMS7w+F075YbHMmz\nuT1J3Kk9ZLE5oIMwTa1arS3QaLLrclg2Gyf4ImCU0heG19LmcdLs1XlJ8EAFRW1l\nLb8wWTwLAULP5Fxnh1vaC33fsCCN/EI=\n-----END EC PRIVATE KEY-----\n" }, { "path": ".docker/clickhouse/single_node_tls/config.xml", "content": "\n\n\n 8443\n 9440\n 0.0.0.0\n\n users.xml\n default\n default\n\n 5368709120\n\n /var/lib/clickhouse/\n /var/lib/clickhouse/tmp/\n /var/lib/clickhouse/user_files/\n /var/lib/clickhouse/access/\n\n \n debug\n /var/log/clickhouse-server/clickhouse-server.log\n /var/log/clickhouse-server/clickhouse-server.err.log\n 1000M\n 10\n 1\n \n\n \n \n /etc/clickhouse-server/certs/server.crt\n /etc/clickhouse-server/certs/server.key\n relaxed\n /etc/clickhouse-server/certs/ca.crt\n true\n sslv2,sslv3,tlsv1\n true\n \n \n\n \n system\n query_log
\n toYYYYMM(event_date)\n 1000\n \n\n \n /var/lib/clickhouse/format_schemas/\n \n \n users.xml\n \n \n\n" }, { "path": ".docker/clickhouse/single_node_tls/users.xml", "content": "\n\n\n \n \n random\n \n \n\n \n \n \n \n ::/0\n \n default\n default\n 1\n \n \n \n client.clickhouseconnect.test\n \n default\n \n \n\n \n \n \n 3600\n 0\n 0\n 0\n 0\n 0\n \n \n \n\n" }, { "path": ".docker/clickhouse/users.xml", "content": "\n\n\n \n \n random\n 1\n \n \n\n \n \n \n \n ::/0\n \n default\n default\n 1\n \n \n foo@bar!\n \n ::/0\n \n default\n default\n 1\n \n \n\n \n \n \n 3600\n 0\n 0\n 0\n 0\n 0\n \n \n \n\n" }, { "path": ".docker/nginx/local.conf", "content": "upstream clickhouse_cluster {\n server clickhouse1:8123;\n server clickhouse2:8123;\n}\n\nserver {\n listen 8123;\n client_max_body_size 100M;\n location / {\n proxy_pass http://clickhouse_cluster;\n }\n}\n" }, { "path": ".docker/setup/Dockerfile", "content": "FROM python:3.11.9-alpine\nCOPY . /app/\nRUN pip install -r /app/requirements.txt\n" }, { "path": ".docker/setup/requirements.txt", "content": "requests==2.32.2\n" }, { "path": ".docker/setup/setup.py", "content": "import copy\nimport logging\nimport os\nimport pprint\n\nimport requests\n\nhost = os.environ.get('host') if os.environ.get('host') else 'http://localhost'\nport = os.environ.get('port') if os.environ.get('port') else '3000'\nadmin_email = os.environ.get('admin_email') if os.environ.get('admin_email') else 'admin@example.com'\nuser_email = os.environ.get('user_email') if os.environ.get('user_email') else 'user@example.com'\npassword = os.environ.get('password') if os.environ.get('password') else 'metabot1'\nsite_name = 'ClickHouse test'\n\nendpoints = {\n 'health_check': '/api/health',\n 'properties': '/api/session/properties',\n 'setup': '/api/setup',\n 'database': '/api/database',\n 'login': '/api/session',\n 'user': '/api/user',\n}\nfor k, v in endpoints.items():\n endpoints[k] = f\"{host}:{port}{v}\"\n\ndb_base_payload = {\n \"is_on_demand\": False,\n \"is_full_sync\": True,\n \"is_sample\": False,\n \"cache_ttl\": None,\n \"refingerprint\": False,\n \"auto_run_queries\": True,\n \"schedules\": {},\n \"details\": {\n \"host\": \"clickhouse\",\n \"port\": 8123,\n \"user\": \"default\",\n \"password\": None,\n \"dbname\": \"default\",\n \"scan-all-databases\": False,\n \"ssl\": False,\n \"tunnel-enabled\": False,\n \"advanced-options\": False\n },\n \"name\": \"Our ClickHouse\",\n \"engine\": \"clickhouse\"\n}\n\n\ndef health():\n response = requests.get(endpoints['health_check'], verify=False)\n if response.json()['status'] == 'ok':\n return 'healthy'\n else:\n health()\n\n\ndef check_response(response, op):\n if response.status_code >= 300:\n print(f'Unexpected status {response.status_code} for {op}', response.text)\n exit(1)\n\n\nif __name__ == '__main__':\n print(\"Checking health\")\n\n if health() == 'healthy' and os.environ.get('retry') is None:\n print(\"Healthy, setting up Metabase\")\n\n session = requests.Session()\n session_token = None\n try:\n token = session.get(endpoints['properties'], verify=False).json()['setup-token']\n setup_payload = {\n 'token': f'{token}',\n 'user': {\n 'first_name': 'Admin',\n 'last_name': 'Admin',\n 'email': admin_email,\n 'site_name': site_name,\n 'password': password,\n 'password_confirm': password\n },\n 'database': None,\n 'invite': None,\n 'prefs': {\n 'site_name': site_name,\n 'site_locale': 'en',\n 'allow_tracking': False\n }\n }\n print(\"Getting the setup token\")\n session_token = session.post(endpoints['setup'], verify=False, json=setup_payload).json()['id']\n except Exception as e:\n print(\"The admin user was already created\")\n\n try:\n if session_token is None:\n session_token = session.post(endpoints['login'], verify=False,\n json={\"username\": admin_email, \"password\": password})\n\n dbs = session.get(endpoints['database'], verify=False).json()\n print(\"Current databases:\")\n pprint.pprint(dbs['data'])\n\n sample_db = next((x for x in dbs['data'] if x['id'] == 1), None)\n if sample_db is not None:\n print(\"Deleting the sample database\")\n res = session.delete(f\"{endpoints['database']}/{sample_db['id']}\")\n check_response(res, 'delete sample db')\n else:\n print(\"The sample database was already deleted\")\n\n single_node_db = next((x for x in dbs['data']\n if x['engine'] == 'clickhouse'\n and x['details']['host'] == 'clickhouse'), None)\n if single_node_db is None:\n print(\"Creating ClickHouse single node db\")\n single_node_payload = copy.deepcopy(db_base_payload)\n single_node_payload['name'] = 'ClickHouse (single node)'\n res = session.post(endpoints['database'], verify=False, json=single_node_payload)\n check_response(res, 'create single node db')\n else:\n print(\"The single node database was already created\")\n\n # cluster_db = next((x for x in dbs['data']\n # if x['engine'] == 'clickhouse'\n # and x['details']['host'] == 'nginx'), None)\n # if cluster_db is None:\n # print(\"Creating ClickHouse cluster db\")\n # cluster_db_payload = copy.deepcopy(db_base_payload)\n # cluster_db_payload['details']['host'] = 'nginx'\n # cluster_db_payload['name'] = 'ClickHouse (cluster)'\n # res = session.post(endpoints['database'], verify=False, json=cluster_db_payload)\n # check_response(res)\n # else:\n # print(\"The cluster database was already created\")\n\n print(\"Creating a regular user\")\n user_payload = {\"first_name\": \"User\", \"last_name\": \"User\", \"email\": user_email, \"password\": password}\n res = session.post(endpoints['user'], verify=False, json=user_payload)\n check_response(res, 'create user')\n\n print(\"Done!\")\n except Exception as e:\n logging.exception(\"Failed to setup Metabase\", e)\n exit()\n" }, { "path": ".github/ISSUE_TEMPLATE/bug_report.md", "content": "---\nname: Bug report\nabout: Create a report to help us improve\ntitle: ''\nlabels: bug\nassignees: ''\n\n---\n\n\n### Describe the bug\n\n### Steps to reproduce\n1.\n2.\n3.\n\n### Expected behaviour\n\n### Error log\n\n### Configuration\n#### Environment\n* metabase-clickhouse-driver version:\n* metabase-clickhouse-driver configuration:\n* Metabase version:\n* OS:\n\n#### ClickHouse server\n* ClickHouse Server version:\n* ClickHouse Server non-default settings, if any:\n* `CREATE TABLE` statements for tables involved:\n* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary\n" }, { "path": ".github/ISSUE_TEMPLATE/feature_request.md", "content": "---\nname: Feature request\nabout: Suggest an idea for the driver\ntitle: ''\nlabels: enhancement\nassignees: ''\n\n---\n\n\n### Use case\n\n### Describe the solution you'd like\n\n### Describe the alternatives you've considered\n\n### Additional context\n" }, { "path": ".github/ISSUE_TEMPLATE/question.md", "content": "---\nname: Question\nabout: Ask a question about the driver\ntitle: ''\nlabels: question\nassignees: ''\n\n---\n\n> Make sure to check the [documentation](https://clickhouse.com/docs/en/integrations/metabase) first.\n> If the question is concise and probably has a short answer,\n> asking it in the [community Slack](https://clickhouse.com/slack) is probably the fastest way to find the answer.\n" }, { "path": ".github/deps.edn", "content": "{:aliases\n {:user/clickhouse\n {:extra-paths [\"PWD/modules/drivers/clickhouse/test\"]\n :extra-deps {metabase/clickhouse {:local/root \"PWD/modules/drivers/clickhouse\"}}}}}\n" }, { "path": ".github/pull_request_template.md", "content": "## Summary\n\n\n## Checklist\nDelete items not relevant to your PR:\n- [ ] Unit and integration tests covering the common scenarios were added\n- [ ] A human-readable description of the changes was provided to include in CHANGELOG\n- [ ] For significant changes, documentation in https://github.com/ClickHouse/clickhouse-docs was updated with further explanations or tutorials\n" }, { "path": ".github/workflows/check-jdbc-snapshot.yml", "content": "# sed -i -E \"s/(com.clickhouse\\/clickhouse-jdbc \\{:mvn\\/version )\\\".+?\\\"\\}/\\1\\\"\\\"}/\" deps.edn\nname: Check with the latest JDBC snapshot\n\non:\n workflow_dispatch:\n inputs:\n jdbc-version:\n description: \"JDBC version to use\"\n required: true\n\nenv:\n # Temporarily using a fork to disable a few failing tests\n METABASE_REPOSITORY: slvrtrn/metabase\n METABASE_VERSION: 0.53.x-ch\n\njobs:\n check-local-current-version:\n runs-on: ubuntu-latest\n steps:\n - name: Checkout Metabase Repo\n uses: actions/checkout@v4\n with:\n repository: ${{ env.METABASE_REPOSITORY }}\n ref: ${{ env.METABASE_VERSION }}\n\n - name: Checkout Driver Repo\n uses: actions/checkout@v4\n with:\n path: modules/drivers/clickhouse\n\n - name: Prepare JDK 21\n uses: actions/setup-java@v4\n with:\n distribution: \"temurin\"\n java-version: \"21\"\n\n - name: Add ClickHouse TLS instance to /etc/hosts\n run: |\n sudo echo \"127.0.0.1 server.clickhouseconnect.test\" | sudo tee -a /etc/hosts\n\n - name: Start ClickHouse in Docker\n uses: hoverkraft-tech/compose-action@v2.0.0\n with:\n compose-file: \"modules/drivers/clickhouse/docker-compose.yml\"\n down-flags: \"--volumes\"\n services: |\n clickhouse\n clickhouse_tls\n clickhouse_older_version\n clickhouse_cluster_node1\n clickhouse_cluster_node2\n nginx\n\n - name: Install Clojure CLI\n run: |\n curl -O https://download.clojure.org/install/linux-install-1.11.1.1182.sh &&\n sudo bash ./linux-install-1.11.1.1182.sh\n\n - name: Setup Node\n uses: actions/setup-node@v4\n with:\n node-version: \"20\"\n cache: \"yarn\"\n\n - name: Update the JDBC version in deps.edn\n working-directory: modules/drivers/clickhouse\n run: |\n sed -i -E \"s/(com.clickhouse\\/clickhouse-jdbc \\{:mvn\\/version )\\\".+?\\\"\\}/\\1\\\"${{ github.event.inputs.jdbc-version }}\\\"}/\" deps.edn\n cat deps.edn\n\n - name: Prepare stuff for pulses\n run: yarn build-static-viz\n\n # Use custom deps.edn containing \"user/clickhouse\" alias to include driver sources\n - name: Prepare deps.edn\n run: |\n mkdir -p /home/runner/.config/clojure\n cat modules/drivers/clickhouse/.github/deps.edn | sed -e \"s|PWD|$PWD|g\" > /home/runner/.config/clojure/deps.edn\n\n - name: Run all tests with the latest ClickHouse version\n env:\n DRIVERS: clickhouse\n run: |\n clojure -X:ci:dev:ee:ee-dev:drivers:drivers-dev:test:user/clickhouse\n" }, { "path": ".github/workflows/check.yml", "content": "name: Check\n\non:\n workflow_dispatch:\n push:\n branches:\n - master\n paths-ignore:\n - '**/*.md'\n pull_request:\n paths-ignore:\n - '**/*.md'\n\nenv:\n # Using a fork to disable a few failing tests\n METABASE_REPOSITORY: slvrtrn/metabase\n METABASE_VERSION: 0.53.x-ch\n\njobs:\n check-local-current-version:\n runs-on: ubuntu-latest\n steps:\n - name: Checkout Metabase Repo\n uses: actions/checkout@v4\n with:\n repository: ${{ env.METABASE_REPOSITORY }}\n ref: ${{ env.METABASE_VERSION }}\n\n - name: Checkout Driver Repo\n uses: actions/checkout@v4\n with:\n path: modules/drivers/clickhouse\n\n - name: Prepare JDK 21\n uses: actions/setup-java@v4\n with:\n distribution: \"temurin\"\n java-version: \"21\"\n\n - name: Add ClickHouse TLS instance to /etc/hosts\n run: |\n sudo echo \"127.0.0.1 server.clickhouseconnect.test\" | sudo tee -a /etc/hosts\n\n - name: Start ClickHouse in Docker\n uses: hoverkraft-tech/compose-action@v2.0.0\n with:\n compose-file: \"modules/drivers/clickhouse/docker-compose.yml\"\n down-flags: \"--volumes\"\n services: |\n clickhouse\n clickhouse_tls\n clickhouse_older_version\n clickhouse_cluster_node1\n clickhouse_cluster_node2\n nginx\n\n - name: Install Clojure CLI\n run: |\n curl -O https://download.clojure.org/install/linux-install-1.11.1.1182.sh &&\n sudo bash ./linux-install-1.11.1.1182.sh\n\n - name: Setup Node\n uses: actions/setup-node@v4\n with:\n node-version: \"20\"\n cache: \"yarn\"\n\n - name: Get M2 cache\n uses: actions/cache@v4\n with:\n path: |\n ~/.m2\n ~/.gitlibs\n key: ${{ runner.os }}-clickhouse-${{ hashFiles('**/deps.edn') }}\n\n - name: Prepare stuff for pulses\n run: yarn build-static-viz\n\n # Use custom deps.edn containing \"user/clickhouse\" alias to include driver sources\n - name: Prepare deps.edn\n run: |\n mkdir -p /home/runner/.config/clojure\n cat modules/drivers/clickhouse/.github/deps.edn | sed -e \"s|PWD|$PWD|g\" > /home/runner/.config/clojure/deps.edn\n\n - name: Run all tests with the latest ClickHouse version\n env:\n DRIVERS: clickhouse\n run: |\n clojure -X:ci:dev:ee:ee-dev:drivers:drivers-dev:test:user/clickhouse\n\n check-local-older-version:\n runs-on: ubuntu-latest\n steps:\n - name: Checkout Metabase Repo\n uses: actions/checkout@v4\n with:\n repository: ${{ env.METABASE_REPOSITORY }}\n ref: ${{ env.METABASE_VERSION }}\n\n - name: Checkout Driver Repo\n uses: actions/checkout@v4\n with:\n path: modules/drivers/clickhouse\n\n - name: Prepare JDK 21\n uses: actions/setup-java@v4\n with:\n distribution: \"temurin\"\n java-version: \"21\"\n\n - name: Add ClickHouse TLS instance to /etc/hosts\n run: |\n sudo echo \"127.0.0.1 server.clickhouseconnect.test\" | sudo tee -a /etc/hosts\n\n - name: Start ClickHouse in Docker\n uses: hoverkraft-tech/compose-action@v2.0.0\n with:\n compose-file: \"modules/drivers/clickhouse/docker-compose.yml\"\n down-flags: \"--volumes\"\n services: |\n clickhouse\n clickhouse_tls\n clickhouse_older_version\n clickhouse_cluster_node1\n clickhouse_cluster_node2\n nginx\n\n - name: Install Clojure CLI\n run: |\n curl -O https://download.clojure.org/install/linux-install-1.11.1.1182.sh &&\n sudo bash ./linux-install-1.11.1.1182.sh\n\n - name: Get M2 cache\n uses: actions/cache@v4\n with:\n path: |\n ~/.m2\n ~/.gitlibs\n key: ${{ runner.os }}-clickhouse-${{ hashFiles('**/deps.edn') }}\n\n # Use custom deps.edn containing \"user/clickhouse\" alias to include driver sources\n - name: Prepare deps.edn\n run: |\n mkdir -p /home/runner/.config/clojure\n cat modules/drivers/clickhouse/.github/deps.edn | sed -e \"s|PWD|$PWD|g\" > /home/runner/.config/clojure/deps.edn\n\n - name: Run ClickHouse driver tests with 23.3\n env:\n DRIVERS: clickhouse\n MB_CLICKHOUSE_TEST_PORT: 8124\n run: |\n clojure -X:ci:dev:ee:ee-dev:drivers:drivers-dev:test:user/clickhouse :only metabase.driver.clickhouse-test\n\n build-jar:\n runs-on: ubuntu-latest\n needs: [ 'check-local-current-version' ]\n steps:\n - name: Checkout Metabase Repo\n uses: actions/checkout@v4\n with:\n repository: ${{ env.METABASE_REPOSITORY }}\n ref: ${{ env.METABASE_VERSION }}\n\n - name: Checkout Driver Repo\n uses: actions/checkout@v4\n with:\n path: modules/drivers/clickhouse\n\n - name: Install Clojure CLI\n run: |\n curl -O https://download.clojure.org/install/linux-install-1.11.1.1182.sh &&\n sudo bash ./linux-install-1.11.1.1182.sh\n\n - name: Build ClickHouse driver\n run: |\n echo \"{:deps {metabase/clickhouse {:local/root \\\"clickhouse\\\" }}}\" > modules/drivers/deps.edn\n bin/build-driver.sh clickhouse\n ls -lah resources/modules\n\n - name: Archive driver JAR\n uses: actions/upload-artifact@v4\n with:\n name: clickhouse.metabase-driver.jar\n path: resources/modules/clickhouse.metabase-driver.jar\n" }, { "path": ".gitignore", "content": "\\#*\\#\n.\\#*\n/target\n*.jar\n*.class\n/.lein-env\n/.lein-failures\n/.lein-plugins\n/.lein-repl-history\n/.clj-kondo\n.eastwood\n.calva\n.cpcache\n.joyride\n.nrepl-port\n.idea\n" }, { "path": "AUTHORS.md", "content": "# Contributors, Authors\n\nThe initial source base comprises major contributions from these authors (_the git log has suffered from frequent brutal rebases, please add yourself here, if I missed you!_):\n\n* Bogdan Mukvich (@Badya)\n* @tsl-karlp\n* Andrew Grigorev (@ei-grad)\n* Felix Mueller (@enqueue)\n" }, { "path": "CHANGELOG.md", "content": "# 1.53.4\n\n### Improvements\n\n* The JDBC driver was updated to [0.8.4](https://github.com/ClickHouse/clickhouse-java/releases/tag/v0.8.4). This fixes [#309](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/309), [#300](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/300), [#297](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/297).\n\n# 1.53.3\n\n### Improvements\n\n* If ClickHouse instance hostname was specified including `http://` or `https://` schema (e.g. `https://sub.example.com`), it will be automatically handled and removed by the driver, instead of failing with a connection error.\n* The JDBC driver was updated to [0.8.2](https://github.com/ClickHouse/clickhouse-java/releases/tag/v0.8.2)\n\n# 1.53.2\n\n### Bug fixes\n\n* The JDBC driver was updated to [0.8.1](https://github.com/ClickHouse/clickhouse-java/releases/tag/v0.8.1) to fix errors in queries with CTEs ([#297](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/297), [#288](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/288), [tadeboro](https://github.com/tadeboro)).\n\n# 1.53.1\n\n### Bug fixes\n\n* Fix unsigned integers overflow ([#293](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/293))\n\n# 1.53.0\n\n### Improvements\n\n* Adds Metabase 0.53.x support ([#287](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/287), [dpsutton](https://github.com/dpsutton)).\n\n### Bug fixes\n\n* Fixed OOB exception on CSV insert caused by an incompatibility with JDBC v2 ([#286](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/286), [wotbrew](https://github.com/wotbrew)).\n\n# 1.52.0\n\n- Formal Metabase 0.52.x+ support\n- The driver now uses JDBC v2 ([0.8.0](https://github.com/ClickHouse/clickhouse-java/releases/tag/v0.8.0))\n- Various improvements to handling of datetimes with timezones\n- `:convert-timezone` feature is disabled for now.\n- Added `max-open-connections` setting under \"advanced options\"; default is 100.\n\n# 1.51.0\n\nAdds Metabase 0.51.x support.\n\n# 1.50.7\n\n### Improvements\n\n* Added a configuration field (under the \"advanced options\", hidden by default) to override certain ClickHouse settings ([#272](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/272)).\n\n# 1.50.6\n\n### Bug fixes\n\n* Fixed null pointer exception on CSV insert ([#268](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/268), [crisptrutski](https://github.com/crisptrutski)).\n\n# 1.50.5\n\n### Bug fixes\n\n* Fixed an error that could occur while setting roles containing hyphens for connection impersonation ([#266](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/266), [sharankow](https://github.com/sharankow)).\n\n# 1.50.4\n\n### Bug fixes\n\n* Fixed an error while uploading a CSV with an offset datetime column ([#263](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/263), [crisptrutski](https://github.com/crisptrutski)).\n\n# 1.50.3\n\n### Improvements\n\n* The driver no longer explicitly sets `allow_experimental_analyzer=0` settings on the connection level; the [new ClickHouse analyzer](https://clickhouse.com/docs/en/operations/analyzer) is now enabled by default.\n\n# 1.50.2\n\n### Bug fixes\n\n* Fixed Array inner type introspection, which could cause reduced performance when querying tables containing arrays. ([#257](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/257))\n\n# 1.50.1\n\n### New features\n\n* Enabled `:set-timezone` ([docs](https://www.metabase.com/docs/latest/configuring-metabase/localization#report-timezone)) Metabase feature in the driver. ([#200](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/200))\n* Enabled `:convert-timezone` ([docs](https://www.metabase.com/docs/latest/questions/query-builder/expressions/converttimezone)) Metabase feature in the driver. ([#254](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/254))\n\n### Other\n\n* The driver now uses [`session_timezone` ClickHouse setting](https://clickhouse.com/docs/en/operations/settings/settings#session_timezone). This is necessary to support the `:set-timezone` and `:convert-timezone` features; however, this setting [was introduced in 23.6](https://clickhouse.com/docs/en/whats-new/changelog/2023#236), which makes it the minimal required ClickHouse version to work with the driver.\n\n# 1.50.0\n\nAfter Metabase 0.50.0, a new naming convention exists for the driver's releases. The new one is intended to reflect the Metabase version the driver is supposed to run on. For example, the driver version 1.50.0 means that it should be used with Metabase v0.50.x or Metabase EE 1.50.x _only_, and it is _not guaranteed_ that this particular version of the driver can work with the previous or the following versions of Metabase.\n\n### New features\n\n* Added Metabase 0.50.x support.\n\n### Improvements\n\n* Bumped the JDBC driver to [0.6.1](https://github.com/ClickHouse/clickhouse-java/releases/tag/v0.6.1).\n\n### Bug fixes\n\n* Fixed the issue where the connection impersonation feature support could be incorrectly reported as disabled.\n\n### Other\n\n* The new ClickHouse analyzer, [which is enabled by default in 24.3+](https://clickhouse.com/blog/clickhouse-release-24-03#analyzer-enabled-by-default), is disabled for the queries executed by the driver, as it shows some compatibilities with the queries generated by Metabase (see [this issue](https://github.com/ClickHouse/ClickHouse/issues/64487) for more details).\n* The `:window-functions/offset` Metabase feature is currently disabled, as the default implementation generates queries incompatible with ClickHouse. See [this issue](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/245) for tracking.\n\n# 1.5.1\n\nMetabase 0.49.14+ only.\n\n### Bug fixes\n\n* Fixed the issue where the Metabase instance could end up broken if the ClickHouse instance was _stopped_ during the upgrade to 1.5.0. ([#242](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/242))\n* Fixed variables substitution with Nullable Date, Date32, DateTime, and DateTime64 columns, where the generated query could fail with NULL values in the database due to an incorrect cast call. ([#243](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/243))\n\n# 1.5.0\n\nMetabase 0.49.14+ only.\n\n### New features\n\n* Added [Metabase CSV Uploads feature](https://www.metabase.com/docs/latest/databases/uploads) support, which is currently enabled with ClickHouse Cloud only. On-premise deployments support will be added in the next release. ([calherries](https://github.com/calherries), [#236](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/236), [#238](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/238))\n* Added [Metabase connection impersonation feature](https://www.metabase.com/learn/permissions/impersonation) support. This feature will be enabled by the driver only if ClickHouse version 24.4+ is detected. ([#219](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/219))\n\n### Improvements\n\n* Proper role setting support on cluster deployments (related issue: [#192](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/192)).\n* Bump the JDBC driver to [0.6.0-patch5](https://github.com/ClickHouse/clickhouse-java/releases/tag/v0.6.0-patch5).\n\n### Bug fixes\n\n* Fixed missing data for the last day when using filters with DateTime columns. ([#202](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/202), [#229](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/229))\n\n# 1.4.0\n\n### New features\n* Metabase 0.49.x support.\n\n### Bug fixes\n* Fixed an incorrect substitution for the current day filter with DateTime columns. ([#216](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/216))\n\n# 1.3.4\n\n### New features\n\n* If introspected ClickHouse version is lower than 23.8, the driver will not use [startsWithUTF8](https://clickhouse.com/docs/en/sql-reference/functions/string-functions#startswithutf8) and fall back to its [non-UTF8 counterpart](https://clickhouse.com/docs/en/sql-reference/functions/string-functions#startswith) instead. There is a drawback in this compatibility mode: potentially incorrect filtering results when working with non-latin strings. If your use case includes filtering by columns with such strings and you experience these issues, consider upgrading your ClickHouse server to 23.8+. ([#224](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/224))\n\n# 1.3.3\n\n### Bug fixes\n* Fixed an issue where it was not possible to create a connection with multiple databases using TLS. ([#215](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/215))\n\n# 1.3.2\n\n### Bug fixes\n* Remove `can-connect?` method override which could cause issues with editing or creating new connections. ([#212](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/212))\n\n\n# 1.3.1\n\n### Bug fixes\n* Fixed incorrect serialization of `Array(UInt8)` columns ([#209](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/209))\n\n# 1.3.0\n\n### New features\n* Metabase 0.48.x support\n\n### Bug fixes\n* Fixed last/next minutes/hours filters with variables creating incorrect queries due to unnecessary `CAST col AS date` call.\n\n# 1.2.5\n\nMetabase 0.47.7+ only.\n\n### New features\n* Added [datetimeDiff](https://www.metabase.com/docs/latest/questions/query-builder/expressions/datetimediff) function support ([#117](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/117))\n\n# 1.2.4\n\nMetabase 0.47.7+ only.\n\n### Bug fixes\n* Fixed UI question -> SQL conversion creating incorrect queries due to superfluous spaces in columns/tables/database names.\n\n# 1.2.3\n\n### Bug fixes\n\n* Fixed `LowCardinality(Nullable)` types introspection, where it was incorrectly reported as `type/*` ([#203](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/203))\n\n# 1.2.2\n\n### Bug fixes\n* Removed forward slash from serialized IPv4/IPv6 columns. NB: IPv4/IPv6 columns are temporarily resolved as `type/TextLike` instead of `type/IPAddress` base type due to an unexpected result in Metabase 0.47 type check.\n* Removed superfluous CAST calls from generated queries that use Date* columns and/or intervals\n\n# 1.2.1\n### New features\n* Use HoneySQL2 in the driver\n\n# 1.2.0\n\n### New features\n* Metabase 0.47 support\n* Connection impersonation support (0.47 feature)\n\n### Bug fixes\n* More correct general database type -> base type mapping\n* `DateTime64` is now correctly mapped to `:type/DateTime`\n* `database-required` field property is now correctly set to `true` if a field is not `Nullable`\n\n# 1.1.7\n\n### New features\n\n* JDBC driver upgrade (v0.4.1 -> [v0.4.6](https://github.com/ClickHouse/clickhouse-java/releases/tag/v0.4.6))\n* Support DateTime64 by [@lucas-tubi](https://github.com/lucas-tubi) ([#165](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/165))\n* Use native `startsWith`/`endsWith` instead of `LIKE str%`/`LIKE %str`\n\n# 1.1.6\n\n### Bug fixes\n\n* Fixed temporal bucketing issues (see [#155](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/155))\n\n# 1.1.5\n\n### Bug fixes\n\n* Fixed Nippy error on cached questions (see [#147](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/147))\n\n# 1.1.4\n\n### Bug fixes\n\n* Fixed `sum-where` behavior where previously it could not be applied to Int columns (see [#156](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/156))\n\n# 1.1.3\n\n### New features\n\n* Hide `.inner` tables of Materialized Views.\n* Resolve `Map` base type to `type/Dictionary`.\n* Database name can now contain multiple schemas in the UI field (space-separated by default), which tells the driver to scan selected databases. Separator can be set in `metabase.driver.clickhouse/SEPARATOR`. (@veschin)\n\n# 1.1.2\n\n### Bug fixes\n\n* Now the driver is able to scan and work with `SimpleAggregateFunction` columns: those were excluded by mistake in 1.0.2.\n\n\n# 1.1.1\n\n### New features\n\n* Metabase 0.46.x compatibility.\n* Added [cljc.java-time](https://clojars.org/com.widdindustries/cljc.java-time) to dependencies, as it is no longer loaded by Metabase.\n\n# 1.1.0\n\n### New features\n\n* Update JDBC driver to v0.4.1.\n* Use new `product_name` additional option instead of `client_name`\n* Replace `sql-jdbc.execute/read-column [:clickhouse Types/ARRAY]` with `sql-jdbc.execute/read-column-thunk [:clickhouse Types/ARRAY]` to be compatible with Metabase 0.46 breaking changes once it is released.\n\n### Bug fixes\n\n* Fix `sql-jdbc.execute/read-column-thunk [:clickhouse Types/TIME]` return type.\n\n# 1.0.4\n\n### New features\n\n* Adds a new \"Scan all databases\" UI toggle (disabled by default), which tells the driver to scan all available databases (excluding `system` and `information_schema`) instead of only the database it is connected to.\n* Database input moved below host/port/username/password in the UI.\n\n# 1.0.3\n\n### Bug fixes\n\n* Fixed NPE that could be thrown by the driver in case of empty database name input.\n\n# 1.0.2\n\n### Bug fixes\n\n* As the underlying JDBC driver version does not support columns with `(Simple)AggregationFunction` type, these columns are now excluded from the table metadata and data browser result sets to prevent sync or data browsing errors.\n\n# 1.0.1\n\n### Bug fixes\n\n* Boolean base type inference fix by [@s-huk](https://github.com/s-huk) (see [#134](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/134))\n\n# 1.0.0\n\nFormal stable release milestone.\n\n### New features\n\n* Added HTTP User-Agent (via clickhouse-jdbc `client_name` setting) with the plugin info according to the [language client spec](https://docs.google.com/document/d/1924Dvy79KXIhfqKpi1EBVY3133pIdoMwgCQtZ-uhEKs/edit#heading=h.ah33hoz5xei2)\n\n# 0.9.2\n\n### New features\n\n* Allow to bypass system-wide proxy settings [#120](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/120)\n\nIt's the first plugin release from the ClickHouse organization.\n\nFrom now on, the plugin is distributed under the Apache 2.0 License.\n\n# 0.9.1\n\n### New features\n\n* Metabase 0.45.x compatibility [#107](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/107)\n* Added SSH tunnel option [#116](https://github.com/ClickHouse/metabase-clickhouse-driver/pull/116)\n\n# 0.9.0\n\n### New features\n\n* Using https://github.com/ClickHouse/clickhouse-jdbc `v0.3.2-patch11`\n\n### Bug fixes\n\n* URLs with underscores [#23](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/23)\n* `now()` timezones issues [#81](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/81)\n* Boolean errors [#88](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/88)\n\nNB: there are messages like this in the Metabase logs\n\n```\n2022-12-07 11:20:58,056 WARN internal.ClickHouseConnectionImpl :: [JDBC Compliant Mode] Transaction is not supported. You may change jdbcCompliant to false to throw SQLException instead.\n2022-12-07 11:20:58,056 WARN internal.ClickHouseConnectionImpl :: [JDBC Compliant Mode] Transaction [ce0e121a-419a-4414-ac39-30f79eff7afd] (0 queries & 0 savepoints) is committed.\n```\n\nUnfortunately, this is the behaviour of the underlying JDBC driver now.\n\nPlease consider raising the log level for `com.clickhouse.jdbc.internal.ClickHouseConnectionImpl` to `ERROR`.\n\n# 0.8.3\n\n### New features\n\n* Enable additional options for ClickHouse connection\n\n# 0.8.2\n\n### New features\n\n* Compatibility with Metabase 0.44\n" }, { "path": "CONTRIBUTING.md", "content": "## Getting started\n\nClickHouse driver for Metabase is an open-source project,\nand we welcome any contributions from the community.\nPlease share your ideas, contribute to the codebase,\nand help us maintain up-to-date documentation.\n\n* Please report any issues you encounter during operations.\n* Feel free to create a pull request, preferably with a test or five.\n\n## Setting up a development environment\n\n### Requirements\n\n* Clojure 1.11+\n* OpenJDK 17\n* Node.js 16.x\n* Yarn\n\nFor testing: Docker Compose\n\nPlease refer to the extensive documentation available on the Metabase website: [Guide to writing a Metabase driver](https://www.metabase.com/docs/latest/developers-guide/drivers/start.html)\n\nClickHouse driver's code should be inside the main Metabase repository checkout in `modules/drivers/clickhouse` directory.\n\nAdditionally, you need to tweak Metabase's `deps.edn` a bit.\n\nThe easiest way to set up a development environment is as follows (mostly the same as in the [CI](https://github.com/enqueue/metabase-clickhouse-driver/blob/master/.github/workflows/check.yml)):\n\n* Clone Metabase and ClickHouse driver repositories\n```bash\ngit clone https://github.com/metabase/metabase.git\ncd metabase\ngit clone https://github.com/enqueue/metabase-clickhouse-driver.git modules/drivers/clickhouse\n```\n\n* Create custom Clojure profiles\n\n```bash\nmkdir -p ~/.clojure\ncat modules/drivers/clickhouse/.github/deps.edn | sed -e \"s|PWD|$PWD|g\" > ~/.clojure/deps.edn\n```\n\nModifying `~/.clojure/deps.edn` will create a new profile: `user/clickhouse`, that adds driver's sources to the class path, and includes all the Metabase tests that are guaranteed to work with the driver.\n\n* Install the Metabase dependencies:\n\n```bash\nclojure -X:deps:drivers prep\n```\n\n* Build the frontend:\n\n```bash\nyarn && yarn build-static-viz\n```\n\n* Add /etc/hosts entry\n\nRequired for TLS tests.\n\n```bash\nsudo -- sh -c \"echo 127.0.0.1 server.clickhouseconnect.test >> /etc/hosts\"\n```\n\n* Start Docker containers\n\n```bash\ndocker compose -f modules/drivers/clickhouse/docker-compose.yml up -d\n```\n\nHere's an overview of the started containers, which have the ports exposed to the `localhost` (see [docker-compose.yml](./docker-compose.yml)):\n\n- Metabase with the ClickHouse driver loaded from the JAR file (port: 3000)\n- Current ClickHouse version (port: 8123) - the main instance for all tests.\n- Current ClickHouse cluster with two nodes (+ nginx as an LB, port: 8127) - required for the set role tests (verifying that the role is set correctly via the query parameters).\n- Current ClickHouse version with TLS support (port: 8443) - required for the TLS tests.\n- Older ClickHouse version (port: 8124) - required for the string functions tests (switch between UTF8 (current) and non-UTF8 (pre-23.8) versions), as well as to verify that certain features, such as connection impersonation, are disabled on the older server versions.\n\nNow, you should be able to run the tests:\n\n```bash\nDRIVERS=clickhouse clojure -X:dev:drivers:drivers-dev:test:user/clickhouse\n```\n\nyou can see that we have our `:user/clickhouse` profile added to the command above, and with `DRIVERS=clickhouse` we instruct Metabase to run the tests only for ClickHouse.\n\nNB: Omitting `DRIVERS` will run the tests for all the built-in database drivers.\n\nIf you want to run tests for only a specific namespace:\n\n```bash\nDRIVERS=clickhouse clojure -X:dev:drivers:drivers-dev:test:user/clickhouse :only metabase.driver.clickhouse-test\n```\n\nor even a single test:\n\n```bash\nDRIVERS=clickhouse clojure -X:dev:drivers:drivers-dev:test:user/clickhouse :only metabase.driver.clickhouse-test/clickhouse-nullable-arrays\n```\n\nTesting the driver with the older ClickHouse version (see [docker-compose.yml](./docker-compose.yml)):\n\n```bash\nMB_CLICKHOUSE_TEST_PORT=8124 DRIVERS=clickhouse clojure -X:dev:drivers:drivers-dev:test:user/clickhouse :only metabase.driver.clickhouse-test\n```\n\n## Building a jar\n\nYou need to add an entry for ClickHouse in `modules/drivers/deps.edn`\n\n```clj\n{:deps\n {...\n metabase/clickhouse {:local/root \"clickhouse\"}\n ...}}\n```\n\nor just run this from the root Metabase directory, overwriting the entire file:\n\n```bash\necho \"{:deps {metabase/clickhouse {:local/root \\\"clickhouse\\\" }}}\" > modules/drivers/deps.edn\n```\n\nNow, you should be able to build the final jar:\n\n```bash\nbin/build-driver.sh clickhouse\n```\n\nAs the result, `resources/modules/clickhouse.metabase-driver.jar` should be created.\n\nFor smoke testing, there is a Metabase with the link to the driver available as a Docker container:\n\n```bash\ndocker compose -f modules/drivers/clickhouse/docker-compose.yml up -d metabase\n```\n\nIt should pick up the driver jar as a volume.\n" }, { "path": "HISTORY.md", "content": "# History\nThe request for a ClickHouse Metabase driver is formulated in [Metabase issue #3332](https://github.com/metabase/metabase/issues/3332). Some impatient ClickHouse users started development. The Metabase team is asking driver developers to publish plug-ins and collect some experiences before considering a PR, so here we are.\n\nThis driver is based on the following PRs:\n* [metabase#8491](https://github.com/metabase/metabase/pull/8491)\n* [metabase#8722](https://github.com/metabase/metabase/pull/8722)\n* [metabase#9469](https://github.com/metabase/metabase/pull/9469)\n" }, { "path": "LICENSE", "content": "Copyright 2016-2025 ClickHouse, Inc.\n\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright 2016-2025 ClickHouse, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "README.md", "content": "

\n\n\n

ClickHouse driver for Metabase

\n

\n
\n

\n\n\n\n

\n\n## About\n\nThe Metabase team promoted the ClickHouse driver to the core level as of Metabase 54 ([release notes](https://github.com/metabase/metabase/releases/tag/v0.54.1), [driver source](https://github.com/metabase/metabase/tree/v0.54.x/modules/drivers/clickhouse)).\n\nFor the end user, this means the following:\n\n- Installing the driver manually is unnecessary, as it is now bundled with Metabase.\n- Starting from April 2025, the Metabase team will continue maintaining the driver. Please report new issues in [the main Metabase repository](https://github.com/metabase/metabase/issues).\n\nThe ClickHouse team recommends avoiding older Metabase versions (53.x and earlier) with manual driver installation; instead, please use the updated Metabase distribution with the driver built-in.\n\n## History\n\nThe request for a ClickHouse Metabase driver was formulated in 2016 in [Metabase issue #3332](https://github.com/metabase/metabase/issues/3332). Several impatient ClickHouse users started the development in the main Metabase repo. In March 2019, after releasing the plugin SDK, the Metabase team [asked to publish the driver separately](https://github.com/metabase/metabase/pull/8491#issuecomment-471721980) in its own repository, and later that month, with Felix Mueller ([@enqueue](https://github.com/enqueue)) leading the efforts, the [initial version of the driver](https://github.com/ClickHouse/metabase-clickhouse-driver/releases/tag/v0.1) was out.\n\nThe original implementation of the driver was based on the following pull requests:\n\n- [metabase#8491](https://github.com/metabase/metabase/pull/8491)\n- [metabase#8722](https://github.com/metabase/metabase/pull/8722)\n- [metabase#9469](https://github.com/metabase/metabase/pull/9469)\n\nThe source base in these PRs comprises major contributions from these authors:\n\n- [@tsl-karlp](https://github.com/tsl-karlp)\n- Andrew Grigorev ([@ei-grad](https://github.com/ei-grad))\n- Bogdan Mukvich ([@Badya](https://github.com/Badya))\n- Felix Mueller ([@enqueue](https://github.com/enqueue))\n\n> [!NOTE]\n> Special thanks to Felix Mueller ([@enqueue](https://github.com/enqueue)), who was the sole maintainer of the project from 2019 to 2022 before transferring it to ClickHouse.\n\nStarting from November 2022, Serge Klochkov ([@slvrtrn](https://github.com/slvrtrn)) joined as a maintainer.\n\nIn early 2023, the repository was transferred to the ClickHouse organization, promoting it as an [official integration](https://clickhouse.com/blog/metabase-clickhouse-plugin-ga-release). Around that time, the driver also became available in [Metabase Cloud](https://www.metabase.com/cloud).\n\nIn April 2025, the driver source code [was moved](https://github.com/metabase/metabase/pull/54740) to the main Metabase repository. Since [Metabase 54](https://github.com/metabase/metabase/releases/tag/v0.54.1), it is now available as a part of the official Metabase bundle.\n" }, { "path": "build_docker_image.sh", "content": "#!/bin/bash\n\nif [ $# -lt 3 ]; then\n echo\n echo \"Usage: ./build_docker_image.sh METABASE_VERSION CLICKHOUSE_DRIVER_VERSION DOCKER_IMAGE_TAG\"\n echo\n echo \"This script builds and tags a Metabase Docker image with ClickHouse driver built-in\"\n echo\n echo \"Example:\"\n echo\n echo \"./build_docker_image.sh v0.44.6 0.8.3 my-metabase-with-clickhouse:v0.0.1\"\n exit 1\nfi\n\nexport DOWNLOAD_URL=\"https://github.com/ClickHouse/metabase-clickhouse-driver/releases/download/$2/clickhouse.metabase-driver.jar\"\necho \"Downloading the driver from $DOWNLOAD_URL\"\n\ncd .build\ncurl -L -o clickhouse.metabase-driver.jar $DOWNLOAD_URL\ndocker build --build-arg METABASE_VERSION=$1 --tag $3 .\nrm clickhouse.metabase-driver.jar\n" }, { "path": "deps.edn", "content": "{:paths [\"src\" \"resources\"]\n :deps\n {com.widdindustries/cljc.java-time {:mvn/version \"0.1.21\"}\n com.clickhouse/clickhouse-jdbc {:mvn/version \"0.8.4\"}\n org.lz4/lz4-java {:mvn/version \"1.8.0\"}}}\n" }, { "path": "docker-compose.yml", "content": "services:\n\n ##########################################################################################################\n # ClickHouse single node (CH driver + Metabase tests)\n ##########################################################################################################\n\n clickhouse:\n image: 'clickhouse/clickhouse-server:25.2-alpine'\n container_name: 'metabase-driver-clickhouse-server'\n hostname: clickhouse\n ports:\n - '8123:8123'\n - '9000:9000'\n environment:\n CLICKHOUSE_SKIP_USER_SETUP: 1\n ulimits:\n nofile:\n soft: 262144\n hard: 262144\n volumes:\n - './.docker/clickhouse/single_node/config.xml:/etc/clickhouse-server/config.xml'\n - './.docker/clickhouse/single_node/users.xml:/etc/clickhouse-server/users.xml'\n\n ##########################################################################################################\n # ClickHouse single node (CH driver TLS tests only)\n ##########################################################################################################\n\n clickhouse_tls:\n build:\n context: ./\n dockerfile: .docker/clickhouse/single_node_tls/Dockerfile\n container_name: 'metabase-driver-clickhouse-server-tls'\n ports:\n - '8443:8443'\n - '9440:9440'\n environment:\n CLICKHOUSE_SKIP_USER_SETUP: 1\n ulimits:\n nofile:\n soft: 262144\n hard: 262144\n volumes:\n - './.docker/clickhouse/single_node_tls/config.xml:/etc/clickhouse-server/config.xml'\n - './.docker/clickhouse/single_node_tls/users.xml:/etc/clickhouse-server/users.xml'\n hostname: server.clickhouseconnect.test\n\n ##########################################################################################################\n # Older ClickHouse version (CH driver tests only)\n # For testing pre-23.8 string functions switch between UTF8 and non-UTF8 versions (see clickhouse_qp.clj)\n ##########################################################################################################\n\n clickhouse_older_version:\n image: 'clickhouse/clickhouse-server:23.3-alpine'\n container_name: 'metabase-driver-clickhouse-server-older-version'\n hostname: clickhouse.older\n ports:\n - '8124:8123'\n - '9001:9000'\n environment:\n CLICKHOUSE_SKIP_USER_SETUP: 1\n ulimits:\n nofile:\n soft: 262144\n hard: 262144\n volumes:\n - './.docker/clickhouse/single_node/config.xml:/etc/clickhouse-server/config.xml'\n - './.docker/clickhouse/single_node/users.xml:/etc/clickhouse-server/users.xml'\n\n ##########################################################################################################\n # ClickHouse cluster (CH driver SET ROLE tests only)\n # See test/metabase/driver/clickhouse_set_role.clj\n ##########################################################################################################\n\n clickhouse_cluster_node1:\n image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-25.2-alpine}'\n ulimits:\n nofile:\n soft: 262144\n hard: 262144\n hostname: clickhouse1\n container_name: metabase-driver-clickhouse-cluster-node-1\n ports:\n - '8125:8123'\n - '9002:9000'\n - '9181:9181'\n environment:\n CLICKHOUSE_SKIP_USER_SETUP: 1\n volumes:\n - './.docker/clickhouse/cluster/server1_config.xml:/etc/clickhouse-server/config.xml'\n - './.docker/clickhouse/cluster/server1_macros.xml:/etc/clickhouse-server/config.d/macros.xml'\n - './.docker/clickhouse/users.xml:/etc/clickhouse-server/users.xml'\n\n clickhouse_cluster_node2:\n image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-25.2-alpine}'\n ulimits:\n nofile:\n soft: 262144\n hard: 262144\n hostname: clickhouse2\n container_name: metabase-driver-clickhouse-cluster-node-2\n ports:\n - '8126:8123'\n - '9003:9000'\n - '9182:9181'\n environment:\n CLICKHOUSE_SKIP_USER_SETUP: 1\n volumes:\n - './.docker/clickhouse/cluster/server2_config.xml:/etc/clickhouse-server/config.xml'\n - './.docker/clickhouse/cluster/server2_macros.xml:/etc/clickhouse-server/config.d/macros.xml'\n - './.docker/clickhouse/users.xml:/etc/clickhouse-server/users.xml'\n\n # Using Nginx as a cluster entrypoint and a round-robin load balancer for HTTP requests\n # See .docker/nginx/local.conf for the configuration\n nginx:\n image: 'nginx:1.23.1-alpine'\n hostname: nginx\n ports:\n - '8127:8123'\n volumes:\n - './.docker/nginx/local.conf:/etc/nginx/conf.d/local.conf'\n container_name: metabase-nginx\n\n ##########################################################################################################\n # Metabase\n ##########################################################################################################\n\n metabase:\n image: metabase/metabase-enterprise:v1.53.6.4\n container_name: metabase-with-clickhouse-driver\n hostname: metabase\n environment:\n 'MB_HTTP_TIMEOUT': '5000'\n 'JAVA_TIMEZONE': 'UTC'\n ports:\n - '3000:3000'\n volumes:\n - '../../../resources/modules/clickhouse.metabase-driver.jar:/plugins/clickhouse.jar'\n - './.docker/clickhouse/single_node_tls/certificates/ca.crt:/certs/ca.crt'\n healthcheck:\n test: curl --fail -X GET -I http://localhost:3000/api/health || exit 1\n interval: 15s\n timeout: 5s\n retries: 10\n\n setup:\n build: .docker/setup/.\n container_name: metabase-clickhouse-setup\n volumes:\n - .docker/setup/setup.py:/app/setup.py\n depends_on:\n metabase:\n condition: service_healthy\n command: python /app/setup.py\n environment:\n host: http://metabase\n port: 3000\n admin_email: 'admin@example.com'\n user_email: 'user@example.com'\n password: 'metabot1'\n" }, { "path": "resources/metabase-plugin.yaml", "content": "info:\n name: Metabase ClickHouse Driver\n version: 1.53.4\n description: Allows Metabase to connect to ClickHouse databases.\ncontact-info:\n name: ClickHouse\n address: https://github.com/ClickHouse/metabase-clickhouse-driver\ndriver:\n name: clickhouse\n display-name: ClickHouse\n lazy-load: true\n parent: sql-jdbc\n connection-properties:\n - host\n - merge:\n - port\n - default: 8123\n - user\n - password\n - name: dbname\n display-name: Databases\n placeholder: default\n helper-text: \"To specify multiple databases, separate them by the space symbol. For example: default data logs.\"\n - name: scan-all-databases\n display-name: Scan all databases\n type: boolean\n default: false\n description: Scan all tables from all available ClickHouse databases except the system ones.\n\n - ssl\n - ssh-tunnel\n - advanced-options-start\n - name: use-no-proxy\n display-name: Disable system wide proxy settings\n default: false\n type: boolean\n visible-if:\n advanced-options: true\n - name: clickhouse-settings\n display-name: ClickHouse settings (comma-separated)\n placeholder: \"allow_experimental_analyzer=1,max_result_rows=100\"\n visible-if:\n advanced-options: true\n - name: max-open-connections\n display-name: \"Max open HTTP connections in the JDBC driver (default: 100)\"\n placeholder: 100\n visible-if:\n advanced-options: true\n - merge:\n - additional-options\n - placeholder: \"connection_timeout=1000&socket_timeout=300000\"\n - default-advanced-options\n connection-properties-include-tunnel-config: true\ninit:\n - step: load-namespace\n namespace: metabase.driver.clickhouse\n - step: register-jdbc-driver\n class: com.clickhouse.jdbc.ClickHouseDriver\n" }, { "path": "src/metabase/driver/clickhouse.clj", "content": "(ns metabase.driver.clickhouse\n \"Driver for ClickHouse databases\"\n #_{:clj-kondo/ignore [:unsorted-required-namespaces]}\n (:require [clojure.core.memoize :as memoize]\n [clojure.string :as str]\n [metabase.config :as config]\n [metabase.driver :as driver]\n [metabase.driver.clickhouse-introspection]\n [metabase.driver.clickhouse-nippy]\n [metabase.driver.clickhouse-qp]\n [metabase.driver.clickhouse-version :as clickhouse-version]\n [metabase.driver.ddl.interface :as ddl.i]\n [metabase.driver.sql :as driver.sql]\n [metabase.driver.sql-jdbc :as sql-jdbc]\n [metabase.driver.sql-jdbc.common :as sql-jdbc.common]\n [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]\n [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]\n [metabase.driver.sql.util :as sql.u]\n [metabase.lib.metadata :as lib.metadata]\n [metabase.query-processor.store :as qp.store]\n [metabase.upload :as upload]\n [metabase.util :as u]\n [metabase.util.log :as log])\n (:import [com.clickhouse.client.api.query QuerySettings]))\n\n(set! *warn-on-reflection* true)\n\n(System/setProperty \"clickhouse.jdbc.v2\" \"true\")\n(driver/register! :clickhouse :parent #{:sql-jdbc})\n\n(defmethod driver/display-name :clickhouse [_] \"ClickHouse\")\n(def ^:private product-name \"metabase/1.53.4\")\n\n(defmethod driver/prettify-native-form :clickhouse\n [_ native-form]\n (sql.u/format-sql-and-fix-params :mysql native-form))\n\n(doseq [[feature supported?] {:standard-deviation-aggregations true\n :now true\n :set-timezone true\n :convert-timezone false\n :test/jvm-timezone-setting false\n :test/date-time-type false\n :test/time-type false\n :schemas true\n :datetime-diff true\n :upload-with-auto-pk false\n :window-functions/offset false\n :window-functions/cumulative (not config/is-test?)\n :left-join (not config/is-test?)\n :describe-fks false\n :actions false\n :metadata/key-constraints (not config/is-test?)}]\n (defmethod driver/database-supports? [:clickhouse feature] [_driver _feature _db] supported?))\n\n(def ^:private default-connection-details\n {:user \"default\" :password \"\" :dbname \"default\" :host \"localhost\" :port \"8123\"})\n\n(defn- connection-details->spec* [details]\n (let [;; ensure defaults merge on top of nils\n details (reduce-kv (fn [m k v] (assoc m k (or v (k default-connection-details))))\n default-connection-details\n details)\n {:keys [user password dbname host port ssl clickhouse-settings max-open-connections]} details\n ;; if multiple databases were specified for the connection,\n ;; use only the first dbname as the \"main\" one\n dbname (first (str/split (str/trim dbname) #\" \"))\n host (cond ; JDBCv1 used to accept schema in the `host` configuration option\n (str/starts-with? host \"http://\") (subs host 7)\n (str/starts-with? host \"https://\") (subs host 8)\n :else host)]\n (->\n {:classname \"com.clickhouse.jdbc.ClickHouseDriver\"\n :subprotocol \"clickhouse\"\n :subname (str \"//\" host \":\" port \"/\" dbname)\n :password (or password \"\")\n :user user\n :ssl (boolean ssl)\n :use_server_time_zone_for_dates true\n :product_name product-name\n :remember_last_set_roles true\n :http_connection_provider \"HTTP_URL_CONNECTION\"\n :jdbc_ignore_unsupported_values \"true\"\n :jdbc_schema_term \"schema\"\n :max_open_connections (or max-open-connections 100)\n ;; see also: https://clickhouse.com/docs/en/integrations/java#configuration\n :custom_http_params (or clickhouse-settings \"\")}\n (sql-jdbc.common/handle-additional-options details :separator-style :url))))\n\n(defmethod sql-jdbc.execute/do-with-connection-with-options :clickhouse\n [driver db-or-id-or-spec {:keys [^String session-timezone _write?] :as options} f]\n (sql-jdbc.execute/do-with-resolved-connection\n driver\n db-or-id-or-spec\n options\n (fn [^java.sql.Connection conn]\n (when-not (sql-jdbc.execute/recursive-connection?)\n (when session-timezone\n (let [^com.clickhouse.jdbc.ConnectionImpl clickhouse-conn (.unwrap conn com.clickhouse.jdbc.ConnectionImpl)\n query-settings (new QuerySettings)]\n (.setOption query-settings \"session_timezone\" session-timezone)\n (.setDefaultQuerySettings clickhouse-conn query-settings)))\n (sql-jdbc.execute/set-best-transaction-level! driver conn)\n (sql-jdbc.execute/set-time-zone-if-supported! driver conn session-timezone)\n (when-let [db (cond\n ;; id?\n (integer? db-or-id-or-spec)\n (qp.store/with-metadata-provider db-or-id-or-spec\n (lib.metadata/database (qp.store/metadata-provider)))\n ;; db?\n (u/id db-or-id-or-spec) db-or-id-or-spec\n ;; otherwise it's a spec and we can't get the db\n :else nil)]\n (sql-jdbc.execute/set-role-if-supported! driver conn db)))\n (f conn))))\n\n(def ^:private ^{:arglists '([db-details])} cloud?\n \"Returns true if the `db-details` are for a ClickHouse Cloud instance, and false otherwise. If it fails to connect\n to the database, it throws a java.sql.SQLException.\"\n (memoize/ttl\n (fn [db-details]\n (let [spec (connection-details->spec* db-details)]\n (sql-jdbc.execute/do-with-connection-with-options\n :clickhouse spec nil\n (fn [^java.sql.Connection conn]\n (with-open [stmt (.createStatement conn)\n rset (.executeQuery stmt \"SELECT value='1' FROM system.settings WHERE name='cloud_mode'\")]\n (if (.next rset) (.getBoolean rset 1) false))))))\n ;; cache the results for 48 hours; TTL is here only to eventually clear out old entries\n :ttl/threshold (* 48 60 60 1000)))\n\n(defmethod sql-jdbc.conn/connection-details->spec :clickhouse\n [_ details]\n (cond-> (connection-details->spec* details)\n (try (cloud? details)\n (catch java.sql.SQLException _e\n false))\n ;; select_sequential_consistency guarantees that we can query data from any replica in CH Cloud\n ;; immediately after it is written\n (assoc :select_sequential_consistency true)))\n\n(defmethod driver/database-supports? [:clickhouse :uploads] [_driver _feature db]\n (if (:details db)\n (try (cloud? (:details db))\n (catch java.sql.SQLException _e\n false))\n false))\n\n(defmethod driver/can-connect? :clickhouse\n [driver details]\n (if config/is-test?\n (try\n ;; Default SELECT 1 is not enough for Metabase test suite,\n ;; as it works slightly differently than expected there\n (let [spec (sql-jdbc.conn/connection-details->spec driver details)\n db (ddl.i/format-name driver (or (:dbname details) (:db details) \"default\"))]\n (sql-jdbc.execute/do-with-connection-with-options\n driver spec nil\n (fn [^java.sql.Connection conn]\n (let [stmt (.prepareStatement conn \"SELECT count(*) > 0 FROM system.databases WHERE name = ?\")\n _ (.setString stmt 1 db)\n rset (.executeQuery stmt)]\n (when (.next rset)\n (.getBoolean rset 1))))))\n (catch Throwable e\n (log/error e \"An exception during ClickHouse connectivity check\")\n false))\n ;; During normal usage, fall back to the default implementation\n (sql-jdbc.conn/can-connect? driver details)))\n\n(defmethod driver/db-default-timezone :clickhouse\n [driver database]\n (sql-jdbc.execute/do-with-connection-with-options\n driver database nil\n (fn [^java.sql.Connection conn]\n (with-open [stmt (.createStatement conn)\n rset (.executeQuery stmt \"SELECT timezone() AS tz\")]\n (when (.next rset)\n (.getString rset 1))))))\n\n(defmethod driver/db-start-of-week :clickhouse [_] :monday)\n\n(defmethod ddl.i/format-name :clickhouse\n [_ table-or-field-name]\n (when table-or-field-name\n (str/replace table-or-field-name #\"-\" \"_\")))\n\n;;; ------------------------------------------ Connection Impersonation ------------------------------------------\n\n(defmethod driver/upload-type->database-type :clickhouse\n [_driver upload-type]\n (case upload-type\n ::upload/varchar-255 \"Nullable(String)\"\n ::upload/text \"Nullable(String)\"\n ::upload/int \"Nullable(Int64)\"\n ::upload/float \"Nullable(Float64)\"\n ::upload/boolean \"Nullable(Boolean)\"\n ::upload/date \"Nullable(Date32)\"\n ::upload/datetime \"Nullable(DateTime64(3))\"\n ::upload/offset-datetime nil))\n\n(defmethod driver/table-name-length-limit :clickhouse\n [_driver]\n ;; FIXME: This is a lie because you're really limited by a filesystems' limits, because Clickhouse uses\n ;; filenames as table/column names. But its an approximation\n 206)\n\n(defn- quote-name [s]\n (let [parts (str/split (name s) #\"\\.\")]\n (str/join \".\" (map #(str \"`\" % \"`\") parts))))\n\n(defn- create-table!-sql\n \"Creates a ClickHouse table with the given name and column definitions. It assumes the engine is MergeTree,\n so it only works with Clickhouse Cloud and single node on-premise deployments at the moment.\"\n [_driver table-name column-definitions & {:keys [primary-key] :as opts}]\n (str/join \"\\n\"\n [(#'sql-jdbc/create-table!-sql :sql-jdbc table-name column-definitions opts)\n \"ENGINE = MergeTree\"\n (format \"ORDER BY (%s)\" (str/join \", \" (map quote-name primary-key)))\n ;; disable insert idempotency to allow duplicate inserts\n \"SETTINGS replicated_deduplication_window = 0\"]))\n\n(defmethod driver/create-table! :clickhouse\n [driver db-id table-name column-definitions & {:keys [primary-key]}]\n (sql-jdbc.execute/do-with-connection-with-options\n driver\n db-id\n {:write? true}\n (fn [^java.sql.Connection conn]\n (with-open [stmt (.createStatement conn)]\n (.execute stmt (create-table!-sql driver table-name column-definitions :primary-key primary-key))))))\n\n(defmethod driver/insert-into! :clickhouse\n [driver db-id table-name column-names values]\n (when (seq values)\n (sql-jdbc.execute/do-with-connection-with-options\n driver\n db-id\n {:write? true}\n (fn [^java.sql.Connection conn]\n (let [sql (format \"INSERT INTO %s (%s) VALUES (%s)\"\n (quote-name table-name)\n (str/join \", \" (map quote-name column-names))\n (str/join \", \" (repeat (count column-names) \"?\")))]\n (with-open [ps (.prepareStatement conn sql)]\n (doseq [row values]\n (when (seq row)\n (doseq [[idx v] (map-indexed (fn [x y] [(inc x) y]) row)]\n (condp isa? (type v)\n nil (.setString ps idx nil)\n java.lang.String (.setString ps idx v)\n java.lang.Boolean (.setBoolean ps idx v)\n java.lang.Long (.setLong ps idx v)\n java.lang.Double (.setFloat ps idx v)\n java.math.BigInteger (.setObject ps idx v)\n java.time.LocalDate (.setObject ps idx v)\n java.time.LocalDateTime (.setObject ps idx v)\n java.time.OffsetDateTime (.setObject ps idx v)\n (.setString ps idx (str v))))\n (.addBatch ps)))\n (doall (.executeBatch ps))))))))\n\n;;; ------------------------------------------ User Impersonation ------------------------------------------\n\n(defmethod driver/database-supports? [:clickhouse :connection-impersonation]\n [_driver _feature db]\n (if db\n (try (clickhouse-version/is-at-least? 24 4 db)\n (catch Throwable _e\n false))\n false))\n\n(defmethod driver.sql/set-role-statement :clickhouse\n [_ role]\n (let [default-role (driver.sql/default-database-role :clickhouse nil)\n quote-if-needed (fn [r]\n (if (or (re-matches #\"\\\".*\\\"\" r) (= role default-role))\n r\n (format \"\\\"%s\\\"\" r)))\n quoted-role (->> (str/split role #\",\")\n (map quote-if-needed)\n (str/join \",\"))\n statement (format \"SET ROLE %s\" quoted-role)]\n statement))\n\n(defmethod driver.sql/default-database-role :clickhouse\n [_ _]\n \"NONE\")\n" }, { "path": "src/metabase/driver/clickhouse_introspection.clj", "content": "(ns metabase.driver.clickhouse-introspection\n (:require [clojure.java.jdbc :as jdbc]\n [clojure.string :as str]\n [metabase.config :as config]\n [metabase.driver :as driver]\n [metabase.driver.ddl.interface :as ddl.i]\n [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]\n [metabase.driver.sql-jdbc.sync :as sql-jdbc.sync]\n [metabase.driver.sql-jdbc.sync.describe-table :as sql-jdbc.describe-table]\n [metabase.util :as u])\n (:import (java.sql DatabaseMetaData)))\n\n(set! *warn-on-reflection* true)\n\n(def ^:private database-type->base-type\n (sql-jdbc.sync/pattern-based-database-type->base-type\n [[#\"array\" :type/Array]\n [#\"bool\" :type/Boolean]\n [#\"date\" :type/Date]\n [#\"date32\" :type/Date]\n [#\"decimal\" :type/Decimal]\n [#\"enum8\" :type/Text]\n [#\"enum16\" :type/Text]\n [#\"fixedstring\" :type/TextLike]\n [#\"float32\" :type/Float]\n [#\"float64\" :type/Float]\n [#\"int8\" :type/Integer]\n [#\"int16\" :type/Integer]\n [#\"int32\" :type/Integer]\n [#\"int64\" :type/BigInteger]\n [#\"ipv4\" :type/IPAddress]\n [#\"ipv6\" :type/IPAddress]\n [#\"map\" :type/Dictionary]\n [#\"string\" :type/Text]\n [#\"tuple\" :type/*]\n [#\"uint8\" :type/Integer]\n [#\"uint16\" :type/Integer]\n [#\"uint32\" :type/Integer]\n [#\"uint64\" :type/BigInteger]\n [#\"uuid\" :type/UUID]]))\n\n(defn- normalize-db-type\n [db-type]\n (cond\n ;; LowCardinality\n (str/starts-with? db-type \"lowcardinality\")\n (normalize-db-type (subs db-type 15 (- (count db-type) 1)))\n ;; Nullable\n (str/starts-with? db-type \"nullable\")\n (normalize-db-type (subs db-type 9 (- (count db-type) 1)))\n ;; for test purposes only: GMT0 is a legacy timezone;\n ;; it maps to LocalDateTime instead of OffsetDateTime\n ;; (= db-type \"datetime64(3, 'gmt0')\")\n ;; :type/DateTime\n ;; DateTime64\n (str/starts-with? db-type \"datetime64\")\n :type/DateTimeWithLocalTZ\n ;; DateTime\n (str/starts-with? db-type \"datetime\")\n :type/DateTimeWithLocalTZ\n ;; Enum*\n (str/starts-with? db-type \"enum\")\n :type/Text\n ;; Map\n (str/starts-with? db-type \"map\")\n :type/Dictionary\n ;; Tuple\n (str/starts-with? db-type \"tuple\")\n :type/*\n ;; SimpleAggregateFunction\n (str/starts-with? db-type \"simpleaggregatefunction\")\n (normalize-db-type (subs db-type (+ (str/index-of db-type \",\") 2) (- (count db-type) 1)))\n ;; _\n :else (or (database-type->base-type (keyword db-type)) :type/*)))\n\n;; Enum8(UInt8) -> :type/Text, DateTime64(Europe/Amsterdam) -> :type/DateTime,\n;; Nullable(DateTime) -> :type/DateTime, SimpleAggregateFunction(sum, Int64) -> :type/BigInteger, etc\n(defmethod sql-jdbc.sync/database-type->base-type :clickhouse\n [_ database-type]\n (let [db-type (if (keyword? database-type)\n (subs (str database-type) 1)\n database-type)]\n (normalize-db-type (u/lower-case-en db-type))))\n\n(defmethod sql-jdbc.sync/excluded-schemas :clickhouse [_]\n #{\"system\" \"information_schema\" \"INFORMATION_SCHEMA\"})\n\n(def ^:private allowed-table-types\n (into-array String\n [\"TABLE\" \"VIEW\" \"FOREIGN TABLE\" \"REMOTE TABLE\" \"DICTIONARY\"\n \"MATERIALIZED VIEW\" \"MEMORY TABLE\" \"LOG TABLE\"]))\n\n(defn- tables-set\n [tables]\n (set\n (for [table tables]\n (let [remarks (:remarks table)]\n {:name (:table_name table)\n :schema (:table_schem table)\n :description (when-not (str/blank? remarks) remarks)}))))\n\n(defn- get-tables-from-metadata\n [^DatabaseMetaData metadata schema-pattern]\n (.getTables metadata\n nil ; catalog - unused in the source code there\n schema-pattern\n \"%\" ; tablePattern \"%\" = match all tables\n allowed-table-types))\n\n(defn- not-inner-mv-table?\n [table]\n (not (str/starts-with? (:table_name table) \".inner\")))\n\n(defn- ->spec\n [db]\n (if (u/id db)\n (sql-jdbc.conn/db->pooled-connection-spec db) db))\n\n(defn- get-all-tables\n [db]\n (jdbc/with-db-metadata [metadata (->spec db)]\n (->> (get-tables-from-metadata metadata \"%\")\n (jdbc/metadata-result)\n (vec)\n (filter #(and\n (not (contains? (sql-jdbc.sync/excluded-schemas :clickhouse) (:table_schem %)))\n (not-inner-mv-table? %)))\n (tables-set))))\n\n;; Strangely enough, the tests only work with :db keyword,\n;; but the actual sync from the UI uses :dbname\n(defn- get-db-name\n [db]\n (or (get-in db [:details :dbname])\n (get-in db [:details :db])))\n\n(defn- get-tables-in-dbs [db-or-dbs]\n (->> (for [db (as-> (or (get-db-name db-or-dbs) \"default\") dbs\n (str/split dbs #\" \")\n (remove empty? dbs)\n (map (comp #(ddl.i/format-name :clickhouse %) str/trim) dbs))]\n (jdbc/with-db-metadata [metadata (->spec db-or-dbs)]\n (jdbc/metadata-result\n (get-tables-from-metadata metadata db))))\n (apply concat)\n (filter not-inner-mv-table?)\n (tables-set)))\n\n(defmethod driver/describe-database :clickhouse\n [_ {{:keys [scan-all-databases]}\n :details :as db}]\n {:tables\n (if\n (boolean scan-all-databases)\n (get-all-tables db)\n (get-tables-in-dbs db))})\n\n(defn- ^:private is-db-required?\n [field]\n (not (str/starts-with? (get field :database-type) \"Nullable\")))\n\n(defmethod driver/describe-table :clickhouse\n [_ database table]\n (let [table-metadata (sql-jdbc.sync/describe-table :clickhouse database table)\n filtered-fields (for [field (:fields table-metadata)\n :let [updated-field (update field :database-required\n (fn [_] (is-db-required? field)))]\n ;; Skip all AggregateFunction (but keeping SimpleAggregateFunction) columns\n ;; JDBC does not support that and it crashes the data browser\n :when (not (re-matches #\"^AggregateFunction\\(.+$\"\n (get field :database-type)))]\n updated-field)]\n (merge table-metadata {:fields (set filtered-fields)})))\n\n(defmethod sql-jdbc.describe-table/get-table-pks :clickhouse\n [_driver ^java.sql.Connection conn db-name-or-nil table]\n ;; JDBC v2 sets the PKs now, so that :metadata/key-constraints feature should be enabled;\n ;; however, enabling :metadata/key-constraints will also enable left-join tests which are currently failing\n (if (not config/is-test?)\n (sql-jdbc.describe-table/get-table-pks :sql-jdbc conn db-name-or-nil table)\n []))\n" }, { "path": "src/metabase/driver/clickhouse_nippy.clj", "content": "(ns metabase.driver.clickhouse-nippy\n (:require [taoensso.nippy :as nippy])\n (:import [java.io DataInput DataOutput]))\n\n(set! *warn-on-reflection* false)\n\n;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;\n;; com.clickhouse.data.value.UnsignedByte\n;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;\n(nippy/extend-freeze com.clickhouse.data.value.UnsignedByte :clickhouse/UnsignedByte\n [^com.clickhouse.data.value.UnsignedByte x ^DataOutput data-output]\n ;; can't enable *warn-on-reflection* because of this call\n (nippy/freeze-to-out! data-output (.toString x)))\n\n(nippy/extend-thaw :clickhouse/UnsignedByte\n [^DataInput data-input]\n (com.clickhouse.data.value.UnsignedByte/valueOf (nippy/thaw-from-in! data-input)))\n\n;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;\n;; com.clickhouse.data.value.UnsignedShort\n;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;\n(nippy/extend-freeze com.clickhouse.data.value.UnsignedShort :clickhouse/UnsignedShort\n [^com.clickhouse.data.value.UnsignedShort x ^DataOutput data-output]\n (nippy/freeze-to-out! data-output (.toString x)))\n\n(nippy/extend-thaw :clickhouse/UnsignedShort\n [^DataInput data-input]\n (com.clickhouse.data.value.UnsignedShort/valueOf (nippy/thaw-from-in! data-input)))\n\n;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;\n;; com.clickhouse.data.value.UnsignedInteger\n;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;\n(nippy/extend-freeze com.clickhouse.data.value.UnsignedInteger :clickhouse/UnsignedInteger\n [^com.clickhouse.data.value.UnsignedInteger x ^DataOutput data-output]\n (nippy/freeze-to-out! data-output (.toString x)))\n\n(nippy/extend-thaw :clickhouse/UnsignedInteger\n [^DataInput data-input]\n (com.clickhouse.data.value.UnsignedInteger/valueOf (nippy/thaw-from-in! data-input)))\n\n;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;\n;; com.clickhouse.data.value.UnsignedLong\n;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;\n(nippy/extend-freeze com.clickhouse.data.value.UnsignedLong :clickhouse/UnsignedLong\n [^com.clickhouse.data.value.UnsignedLong x ^DataOutput data-output]\n (nippy/freeze-to-out! data-output (.toString x)))\n\n(nippy/extend-thaw :clickhouse/UnsignedLong\n [^DataInput data-input]\n (com.clickhouse.data.value.UnsignedLong/valueOf (nippy/thaw-from-in! data-input)))\n" }, { "path": "src/metabase/driver/clickhouse_qp.clj", "content": "(ns metabase.driver.clickhouse-qp\n \"CLickHouse driver: QueryProcessor-related definition\"\n #_{:clj-kondo/ignore [:unsorted-required-namespaces]}\n (:require [clojure.string :as str]\n [honey.sql :as sql]\n [java-time.api :as t]\n [metabase.driver.clickhouse-nippy]\n [metabase.driver.clickhouse-version :as clickhouse-version]\n [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]\n [metabase.driver.sql.query-processor :as sql.qp :refer [add-interval-honeysql-form]]\n [metabase.driver.sql.util :as sql.u]\n [metabase.driver.sql.util.unprepare :as unprepare]\n [metabase.legacy-mbql.util :as mbql.u]\n [metabase.query-processor.timezone :as qp.timezone]\n [metabase.util :as u]\n [metabase.util.date-2 :as u.date]\n [metabase.util.honey-sql-2 :as h2x])\n (:import [java.sql ResultSet ResultSetMetaData Types]\n [java.time\n LocalDate\n LocalDateTime\n LocalTime\n OffsetDateTime\n OffsetTime\n ZonedDateTime]\n java.util.Arrays))\n\n;; (set! *warn-on-reflection* true) ;; isn't enabled because of Arrays/toString call\n\n(defmethod sql.qp/quote-style :clickhouse [_] :mysql)\n\n;; without try, there might be test failures when QP is not yet initialized\n;; e.g., when a test is preparing the dataset\n(defn- get-report-timezone-id-safely\n []\n (try\n (qp.timezone/report-timezone-id-if-supported)\n (catch Throwable _e nil)))\n\n;; datetime('europe/amsterdam') -> europe/amsterdam\n(defn- extract-datetime-timezone\n [db-type]\n (when (and db-type (string? db-type))\n (cond\n ;; e.g. DateTime64(3, 'Europe/Amsterdam')\n (str/starts-with? db-type \"datetime64\")\n (if (> (count db-type) 17) (subs db-type 15 (- (count db-type) 2)) nil)\n ;; e.g. DateTime('Europe/Amsterdam')\n (str/starts-with? db-type \"datetime\")\n (if (> (count db-type) 12) (subs db-type 10 (- (count db-type) 2)) nil)\n ;; _\n :else nil)))\n\n(defn- remove-low-cardinality-and-nullable\n [db-type]\n (when (and db-type (string? db-type))\n (let [db-type-lowercase (u/lower-case-en db-type)\n without-low-car (if (str/starts-with? db-type-lowercase \"lowcardinality(\")\n (subs db-type-lowercase 15 (- (count db-type-lowercase) 1))\n db-type-lowercase)\n without-nullable (if (str/starts-with? without-low-car \"nullable(\")\n (subs without-low-car 9 (- (count without-low-car) 1))\n without-low-car)]\n without-nullable)))\n\n(defn- in-report-timezone\n [expr]\n (let [report-timezone (get-report-timezone-id-safely)\n lower (u/lower-case-en (h2x/database-type expr))\n db-type (remove-low-cardinality-and-nullable lower)]\n (if (and report-timezone (string? db-type) (str/starts-with? db-type \"datetime\"))\n (let [timezone (extract-datetime-timezone db-type)]\n (if (not (= timezone (u/lower-case-en report-timezone)))\n [:'toTimeZone expr (h2x/literal report-timezone)]\n expr))\n expr)))\n\n(defmethod sql.qp/date [:clickhouse :default]\n [_ _ expr]\n expr)\n\n;;; ------------------------------------------------------------------------------------\n;;; Extract functions\n;;; ------------------------------------------------------------------------------------\n\n(defn- date-extract\n [ch-fn expr db-type]\n (-> [ch-fn (in-report-timezone expr)]\n (h2x/with-database-type-info db-type)))\n\n(defmethod sql.qp/date [:clickhouse :day-of-week]\n [_ _ expr]\n ;; a tick in the function name prevents HSQL2 to make the function call UPPERCASE\n ;; https://cljdoc.org/d/com.github.seancorfield/honeysql/2.4.1011/doc/getting-started/other-databases#clickhouse\n (sql.qp/adjust-day-of-week\n :clickhouse (date-extract :'toDayOfWeek expr \"uint8\")))\n\n(defmethod sql.qp/date [:clickhouse :month-of-year]\n [_ _ expr]\n (date-extract :'toMonth expr \"uint8\"))\n\n(defmethod sql.qp/date [:clickhouse :minute-of-hour]\n [_ _ expr]\n (date-extract :'toMinute expr \"uint8\"))\n\n(defmethod sql.qp/date [:clickhouse :hour-of-day]\n [_ _ expr]\n (date-extract :'toHour expr \"uint8\"))\n\n(defmethod sql.qp/date [:clickhouse :day-of-month]\n [_ _ expr]\n (date-extract :'toDayOfMonth expr \"uint8\"))\n\n(defmethod sql.qp/date [:clickhouse :day-of-year]\n [_ _ expr]\n (date-extract :'toDayOfYear expr \"uint16\"))\n\n(defmethod sql.qp/date [:clickhouse :week-of-year-iso]\n [_ _ expr]\n (date-extract :'toISOWeek expr \"uint8\"))\n\n(defmethod sql.qp/date [:clickhouse :quarter-of-year]\n [_ _ expr]\n (date-extract :'toQuarter expr \"uint8\"))\n\n(defmethod sql.qp/date [:clickhouse :year-of-era]\n [_ _ expr]\n (date-extract :'toYear expr \"uint16\"))\n\n;;; ------------------------------------------------------------------------------------\n;;; Truncate functions\n;;; ------------------------------------------------------------------------------------\n\n(defn- date-trunc\n [ch-fn expr]\n [ch-fn (in-report-timezone expr)])\n\n(defn- to-start-of-week\n [expr]\n (date-trunc :'toMonday expr))\n\n(defmethod sql.qp/date [:clickhouse :minute]\n [_ _ expr]\n (date-trunc :'toStartOfMinute expr))\n\n(defmethod sql.qp/date [:clickhouse :hour]\n [_ _ expr]\n (date-trunc :'toStartOfHour expr))\n\n(defmethod sql.qp/date [:clickhouse :day]\n [_ _ expr]\n (date-trunc :'toStartOfDay expr))\n\n(defmethod sql.qp/date [:clickhouse :week]\n [driver _ expr]\n (sql.qp/adjust-start-of-week driver to-start-of-week expr))\n\n(defmethod sql.qp/date [:clickhouse :month]\n [_ _ expr]\n (date-trunc :'toStartOfMonth expr))\n\n(defmethod sql.qp/date [:clickhouse :quarter]\n [_ _ expr]\n (date-trunc :'toStartOfQuarter expr))\n\n(defmethod sql.qp/date [:clickhouse :year]\n [_ _ expr]\n (date-trunc :'toStartOfYear expr))\n\n;;; ------------------------------------------------------------------------------------\n;;; Unix timestamps functions\n;;; ------------------------------------------------------------------------------------\n\n(defmethod sql.qp/unix-timestamp->honeysql [:clickhouse :seconds]\n [_ _ expr]\n (h2x/->datetime expr))\n\n(defmethod sql.qp/unix-timestamp->honeysql [:clickhouse :milliseconds]\n [_ _ expr]\n (let [report-timezone (get-report-timezone-id-safely)\n inner-expr (h2x// expr 1000)]\n (if report-timezone\n [:'toDateTime64 inner-expr 3 report-timezone]\n [:'toDateTime64 inner-expr 3])))\n\n(defmethod sql.qp/unix-timestamp->honeysql [:clickhouse :microseconds]\n [_ _ expr]\n (let [report-timezone (get-report-timezone-id-safely)\n inner-expr [:'toInt64 (h2x// expr 1000)]]\n (if report-timezone\n [:'fromUnixTimestamp64Milli inner-expr report-timezone]\n [:'fromUnixTimestamp64Milli inner-expr])))\n\n;;; ------------------------------------------------------------------------------------\n;;; HoneySQL forms\n;;; ------------------------------------------------------------------------------------\n\n(defmethod sql.qp/->honeysql [:clickhouse :convert-timezone]\n [driver [_ arg target-timezone source-timezone]]\n (let [expr (sql.qp/->honeysql driver (cond-> arg (string? arg) u.date/parse))\n with-tz-info? (h2x/is-of-type? expr #\"(?:nullable\\(|lowcardinality\\()?(datetime64\\(\\d, {0,1}'.*|datetime\\(.*)\")\n _ (sql.u/validate-convert-timezone-args with-tz-info? target-timezone source-timezone)]\n (if (not with-tz-info?)\n [:'plus\n expr\n [:'toIntervalSecond\n [:'minus\n [:'timeZoneOffset [:'toTimeZone expr target-timezone]]\n [:'timeZoneOffset [:'toTimeZone expr source-timezone]]]]]\n [:'toTimeZone expr target-timezone])))\n\n(defmethod sql.qp/current-datetime-honeysql-form :clickhouse\n [_]\n (let [report-timezone (get-report-timezone-id-safely)\n [expr db-type] (if report-timezone\n [[:'now64 [:raw 9] (h2x/literal report-timezone)] (format \"DateTime64(9, '%s')\" report-timezone)]\n [[:'now64 [:raw 9]] \"DateTime64(9)\"])]\n (h2x/with-database-type-info expr db-type)))\n\n(defn- date-time-parse-fn\n [nano]\n (if (zero? nano) :'parseDateTimeBestEffort :'parseDateTime64BestEffort))\n\n(defmethod sql.qp/->honeysql [:clickhouse LocalDateTime]\n [_ ^java.time.LocalDateTime t]\n (let [formatted (t/format \"yyyy-MM-dd HH:mm:ss.SSS\" t)\n report-tz (or (get-report-timezone-id-safely) \"UTC\")]\n (if (zero? (.getNano t))\n [:'parseDateTimeBestEffort formatted report-tz]\n [:'parseDateTime64BestEffort formatted 3 report-tz])))\n\n(defmethod sql.qp/->honeysql [:clickhouse ZonedDateTime]\n [_ ^java.time.ZonedDateTime t]\n (let [formatted (t/format \"yyyy-MM-dd HH:mm:ss.SSSZZZZZ\" t)\n fn (date-time-parse-fn (.getNano t))]\n [fn formatted]))\n\n(defmethod sql.qp/->honeysql [:clickhouse OffsetDateTime]\n [_ ^java.time.OffsetDateTime t]\n ;; copy-paste due to reflection warnings\n (let [formatted (t/format \"yyyy-MM-dd HH:mm:ss.SSSZZZZZ\" t)\n fn (date-time-parse-fn (.getNano t))]\n [fn formatted]))\n\n(defmethod sql.qp/->honeysql [:clickhouse LocalDate]\n [_ ^java.time.LocalDate t]\n [:'parseDateTimeBestEffort t])\n\n(defn- local-date-time\n [^java.time.LocalTime t]\n (t/local-date-time (t/local-date 1970 1 1) t))\n\n(defmethod sql.qp/->honeysql [:clickhouse LocalTime]\n [driver ^java.time.LocalTime t]\n (sql.qp/->honeysql driver (local-date-time t)))\n\n(defmethod sql.qp/->honeysql [:clickhouse OffsetTime]\n [driver ^java.time.OffsetTime t]\n (sql.qp/->honeysql driver (t/offset-date-time\n (local-date-time (.toLocalTime t))\n (.getOffset t))))\n\n(defn- args->float64\n [args]\n (map (fn [arg] [:'toFloat64 (sql.qp/->honeysql :clickhouse arg)]) args))\n\n(defn- interval? [expr]\n (mbql.u/is-clause? :interval expr))\n\n(defmethod sql.qp/->honeysql [:clickhouse :+]\n [driver [_ & args]]\n (if (some interval? args)\n (if-let [[field intervals] (u/pick-first (complement interval?) args)]\n (reduce (fn [hsql-form [_ amount unit]]\n (add-interval-honeysql-form driver hsql-form amount unit))\n (sql.qp/->honeysql driver field)\n intervals)\n (throw (ex-info \"Summing intervals is not supported\" {:args args})))\n (into [:+] (args->float64 args))))\n\n(defmethod sql.qp/->honeysql [:clickhouse :log]\n [driver [_ field]]\n [:'log10 (sql.qp/->honeysql driver field)])\n\n(defn- format-expr\n [expr]\n (first (sql/format-expr (sql.qp/->honeysql :clickhouse expr) {:nested true})))\n\n(defmethod sql.qp/->honeysql [:clickhouse :percentile]\n [_ [_ field p]]\n [:raw (format \"quantile(%s)(%s)\" (format-expr p) (format-expr field))])\n\n(defmethod sql.qp/->honeysql [:clickhouse :regex-match-first]\n [driver [_ arg pattern]]\n [:'extract (sql.qp/->honeysql driver arg) pattern])\n\n(defmethod sql.qp/->honeysql [:clickhouse :stddev]\n [driver [_ field]]\n [:'stddevPop (sql.qp/->honeysql driver field)])\n\n(defmethod sql.qp/->honeysql [:clickhouse :median]\n [driver [_ field]]\n [:'median (sql.qp/->honeysql driver field)])\n\n;; Substring does not work for Enums, so we need to cast to String\n(defmethod sql.qp/->honeysql [:clickhouse :substring]\n [driver [_ arg start length]]\n (let [str [:'toString (sql.qp/->honeysql driver arg)]]\n (if length\n [:'substring str\n (sql.qp/->honeysql driver start)\n (sql.qp/->honeysql driver length)]\n [:'substring str\n (sql.qp/->honeysql driver start)])))\n\n(defmethod sql.qp/->honeysql [:clickhouse :var]\n [driver [_ field]]\n [:'varPop (sql.qp/->honeysql driver field)])\n\n(defmethod sql.qp/->float :clickhouse\n [_ value]\n [:'toFloat64 value])\n\n(defmethod sql.qp/->honeysql [:clickhouse :value]\n [driver value]\n (let [[_ value {base-type :base_type}] value]\n (when (some? value)\n (condp #(isa? %2 %1) base-type\n :type/IPAddress [:'toIPv4 value]\n (sql.qp/->honeysql driver value)))))\n\n(defmethod sql.qp/->honeysql [:clickhouse :=]\n [driver [op field value]]\n (let [[qual valuevalue fieldinfo] value\n hsql-field (sql.qp/->honeysql driver field)\n hsql-value (sql.qp/->honeysql driver value)]\n (if (and (isa? qual :value)\n (isa? (:base_type fieldinfo) :type/Text)\n (nil? valuevalue))\n [:or\n [:= hsql-field hsql-value]\n [:= [:'empty hsql-field] 1]]\n ((get-method sql.qp/->honeysql [:sql :=]) driver [op field value]))))\n\n(defmethod sql.qp/->honeysql [:clickhouse :!=]\n [driver [op field value]]\n (let [[qual valuevalue fieldinfo] value\n hsql-field (sql.qp/->honeysql driver field)\n hsql-value (sql.qp/->honeysql driver value)]\n (if (and (isa? qual :value)\n (isa? (:base_type fieldinfo) :type/Text)\n (nil? valuevalue))\n [:and\n [:!= hsql-field hsql-value]\n [:= [:'notEmpty hsql-field] 1]]\n ((get-method sql.qp/->honeysql [:sql :!=]) driver [op field value]))))\n\n;; I do not know why the tests expect nil counts for empty results\n;; but that's how it is :-)\n;;\n;; It would even be better if we could use countIf and sumIf directly\n;;\n;; metabase.query-processor-test.count-where-test\n;; metabase.query-processor-test.share-test\n(defmethod sql.qp/->honeysql [:clickhouse :count-where]\n [driver [_ pred]]\n [:case\n [:> [:'count] 0]\n [:sum [:case (sql.qp/->honeysql driver pred) 1 :else 0]]\n :else nil])\n\n(defmethod sql.qp/->honeysql [:clickhouse :sum-where]\n [driver [_ field pred]]\n [:sum [:case (sql.qp/->honeysql driver pred) (sql.qp/->honeysql driver field)\n :else 0]])\n\n(defmethod sql.qp/add-interval-honeysql-form :clickhouse\n [_ dt amount unit]\n (h2x/+ dt [:raw (format \"INTERVAL %d %s\" (int amount) (name unit))]))\n\n(defn- clickhouse-string-fn\n [fn-name field value options]\n (let [hsql-field (sql.qp/->honeysql :clickhouse field)\n hsql-value (sql.qp/->honeysql :clickhouse value)]\n (if (get options :case-sensitive true)\n [fn-name hsql-field hsql-value]\n [fn-name [:'lowerUTF8 hsql-field] [:'lowerUTF8 hsql-value]])))\n\n(defmethod sql.qp/->honeysql [:clickhouse :starts-with]\n [_ [_ field value options]]\n (let [starts-with (clickhouse-version/with-min 23 8\n (constantly :'startsWithUTF8)\n (constantly :'startsWith))]\n (clickhouse-string-fn starts-with field value options)))\n\n(defmethod sql.qp/->honeysql [:clickhouse :ends-with]\n [_ [_ field value options]]\n (let [ends-with (clickhouse-version/with-min 23 8\n (constantly :'endsWithUTF8)\n (constantly :'endsWith))]\n (clickhouse-string-fn ends-with field value options)))\n\n(defmethod sql.qp/->honeysql [:clickhouse :contains]\n [_ [_ field value options]]\n (let [hsql-field (sql.qp/->honeysql :clickhouse field)\n hsql-value (sql.qp/->honeysql :clickhouse value)\n position-fn (if (get options :case-sensitive true)\n :'positionUTF8\n :'positionCaseInsensitiveUTF8)]\n [:> [position-fn hsql-field hsql-value] 0]))\n\n(defmethod sql.qp/->honeysql [:clickhouse :datetime-diff]\n [driver [_ x y unit]]\n (let [x (sql.qp/->honeysql driver x)\n y (sql.qp/->honeysql driver y)]\n (case unit\n ;; Week: Metabase tests expect a bit different result from what `age` provides\n (:week)\n [:'intDiv [:'dateDiff (h2x/literal :day) (date-trunc :'toStartOfDay x) (date-trunc :'toStartOfDay y)] [:raw 7]]\n ;; -------------------------\n (:year :month :quarter :day)\n [:'age (h2x/literal unit) (date-trunc :'toStartOfDay x) (date-trunc :'toStartOfDay y)]\n ;; -------------------------\n (:hour :minute :second)\n [:'age (h2x/literal unit) (in-report-timezone x) (in-report-timezone y)])))\n\n;; We do not have Time data types, so we cheat a little bit\n(defmethod sql.qp/cast-temporal-string [:clickhouse :Coercion/ISO8601->Time]\n [_driver _special_type expr]\n [:'parseDateTimeBestEffort [:'concat \"1970-01-01T\" expr]])\n\n(defmethod sql.qp/cast-temporal-byte [:clickhouse :Coercion/ISO8601->Time]\n [_driver _special_type expr]\n expr)\n\n;;; ------------------------------------------------------------------------------------\n;;; JDBC-related functions\n;;; ------------------------------------------------------------------------------------\n\n(defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/TINYINT]\n [_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i]\n (fn []\n (.getObject rs i)))\n\n(defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/SMALLINT]\n [_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i]\n (fn []\n (.getObject rs i)))\n\n;; This is for tests only - some of them expect nil values\n;; getInt/getLong return 0 in case of a NULL value in the result set\n;; the only way to check if it was actually NULL - call ResultSet.wasNull afterwards\n(defn- with-null-check\n [^ResultSet rs value]\n (if (.wasNull rs) nil value))\n\n(defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/BIGINT]\n [_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i]\n (fn []\n (with-null-check rs (.getBigDecimal rs i))))\n\n(defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/INTEGER]\n [_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i]\n (fn []\n (with-null-check rs (.getLong rs i))))\n\n(def ^:private utc-zone-id (java.time.ZoneId/of \"UTC\"))\n(defn- zdt-in-report-timezone\n [^ZonedDateTime zdt]\n (let [maybe-report-timezone (get-report-timezone-id-safely)]\n (if maybe-report-timezone\n (.withZoneSameInstant zdt (java.time.ZoneId/of maybe-report-timezone))\n (if (= (.getId (.getZone zdt)) \"GMT0\") ;; for test purposes only; GMT0 is a legacy tz\n (.withZoneSameInstant zdt utc-zone-id)\n zdt))))\n\n(defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/DATE]\n [_ ^ResultSet rs ^ResultSetMetaData _rsmeta ^Integer i]\n (fn []\n (when-let [sql-date (.getDate rs i)]\n (.toLocalDate sql-date))))\n\n(defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/TIMESTAMP]\n [_ ^ResultSet rs ^ResultSetMetaData rsmeta ^Integer i]\n (fn []\n (when-let [zdt (.getObject rs i ZonedDateTime)]\n (let [db-type (remove-low-cardinality-and-nullable (.getColumnTypeName rsmeta i))]\n (if (= db-type \"datetime64(3, 'gmt0')\")\n ;; a hack for some MB test assertions only; GMT0 is a legacy tz\n (.toLocalDateTime (zdt-in-report-timezone zdt))\n ;; this is the normal behavior\n (.toOffsetDateTime (.withZoneSameInstant\n (zdt-in-report-timezone zdt)\n utc-zone-id)))))))\n\n(defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/TIME]\n [_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i]\n (fn []\n (.getObject rs i OffsetTime)))\n\n(defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/NUMERIC]\n [_ ^ResultSet rs ^ResultSetMetaData rsmeta ^Integer i]\n (fn []\n ; count is NUMERIC cause UInt64 is too large for the canonical SQL BIGINT,\n ; and defaults to BigDecimal, but we want it to be coerced to java Long\n ; cause it still fits and the tests are expecting that\n (if (= (.getColumnLabel rsmeta i) \"count\")\n (.getLong rs i)\n (.getBigDecimal rs i))))\n\n(defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/ARRAY]\n [_ ^ResultSet rs ^ResultSetMetaData _rsmeta ^Integer i]\n (fn []\n (when-let [arr (.getArray rs i)]\n (Arrays/deepToString (.getArray arr)))))\n\n(defn- ipv4-column->string\n [^ResultSet rs ^Integer i]\n (when-let [inet-address (.getObject rs i java.net.Inet4Address)]\n (.getHostAddress inet-address)))\n\n(defn- ipv6-column->string\n [^ResultSet rs ^Integer i]\n (when-let [inet-address (.getObject rs i java.net.Inet6Address)]\n (.getHostAddress inet-address)))\n\n(defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/OTHER]\n [_ ^ResultSet rs ^ResultSetMetaData rsmeta ^Integer i]\n (fn []\n (let [normalized-db-type (remove-low-cardinality-and-nullable\n (.getColumnTypeName rsmeta i))]\n (cond\n (= normalized-db-type \"ipv4\")\n (ipv4-column->string rs i)\n (= normalized-db-type \"ipv6\")\n (ipv6-column->string rs i)\n ;; _\n :else (.getObject rs i)))))\n\n(defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/VARCHAR]\n [_ ^ResultSet rs ^ResultSetMetaData rsmeta ^Integer i]\n (fn []\n (let [normalized-db-type (remove-low-cardinality-and-nullable\n (.getColumnTypeName rsmeta i))]\n (cond\n ;; Enum8/Enum16\n (str/starts-with? normalized-db-type \"enum\")\n (.getString rs i)\n ;; _\n :else (.getObject rs i)))))\n\n(defmethod unprepare/unprepare-value [:clickhouse LocalDate]\n [_ t]\n (format \"'%s'\" (t/format \"yyyy-MM-dd\" t)))\n\n(defmethod unprepare/unprepare-value [:clickhouse LocalTime]\n [_ t]\n (format \"'%s'\" (t/format \"HH:mm:ss.SSS\" t)))\n\n(defmethod unprepare/unprepare-value [:clickhouse OffsetTime]\n [_ t]\n (format \"'%s'\" (t/format \"HH:mm:ss.SSSZZZZZ\" t)))\n\n(defmethod unprepare/unprepare-value [:clickhouse LocalDateTime]\n [_ t]\n (format \"'%s'\" (t/format \"yyyy-MM-dd HH:mm:ss.SSS\" t)))\n\n(defmethod unprepare/unprepare-value [:clickhouse OffsetDateTime]\n [_ ^OffsetDateTime t]\n (format \"%s('%s')\"\n (if (zero? (.getNano t)) \"parseDateTimeBestEffort\" \"parseDateTime64BestEffort\")\n (t/format \"yyyy-MM-dd HH:mm:ss.SSSZZZZZ\" t)))\n\n(defmethod unprepare/unprepare-value [:clickhouse ZonedDateTime]\n [_ t]\n (format \"'%s'\" (t/format \"yyyy-MM-dd HH:mm:ss.SSSZZZZZ\" t)))\n" }, { "path": "src/metabase/driver/clickhouse_version.clj", "content": "\"Provides the info about the ClickHouse version. Extracted from the main clickhouse.clj file,\n as both Driver and QP overrides require access to it, avoiding circular dependencies.\"\n(ns metabase.driver.clickhouse-version\n (:require [clojure.core.memoize :as memoize]\n [metabase.driver :as driver]\n [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]\n [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]\n [metabase.driver.util :as driver.u]\n [metabase.lib.metadata :as lib.metadata]\n [metabase.query-processor.store :as qp.store]))\n\n(set! *warn-on-reflection* true)\n\n;; cache the results for 60 minutes;\n;; TTL is here only to eventually clear out old entries/keep it from growing too large\n(def ^:private default-cache-ttl (* 60 60 1000))\n\n(def ^:private clickhouse-version-query\n (str \"WITH s AS (SELECT version() AS ver, splitByChar('.', ver) AS verSplit) \"\n \"SELECT s.ver, toInt32(verSplit[1]), toInt32(verSplit[2]) FROM s\"))\n\n(def ^:private ^{:arglists '([db-details])} get-clickhouse-version\n (memoize/ttl\n (fn [db-details]\n (sql-jdbc.execute/do-with-connection-with-options\n :clickhouse\n (sql-jdbc.conn/connection-details->spec :clickhouse db-details)\n nil\n (fn [^java.sql.Connection conn]\n (with-open [stmt (.createStatement conn)\n rset (.executeQuery stmt clickhouse-version-query)]\n (when (.next rset)\n {:version (.getString rset 1)\n :semantic-version {:major (.getInt rset 2)\n :minor (.getInt rset 3)}})))))\n :ttl/threshold default-cache-ttl))\n\n(defmethod driver/dbms-version :clickhouse\n [_driver db]\n (get-clickhouse-version (:details db)))\n\n(defn is-at-least?\n \"Is ClickHouse version at least `major.minor` (e.g., 24.4)?\"\n ([major minor]\n ;; used from the QP overrides; we don't have access to the DB object\n (is-at-least? major minor (lib.metadata/database (qp.store/metadata-provider))))\n ([major minor db]\n ;; used from the Driver overrides; we have access to the DB object\n (let [version (driver/dbms-version :clickhouse db)\n semantic (:semantic-version version)]\n (driver.u/semantic-version-gte [(:major semantic) (:minor semantic)] [major minor]))))\n\n(defn with-min\n \"Execute `f` if the ClickHouse version is greater or equal to `major.minor` (e.g., 24.4);\n otherwise, execute `fallback-f`, if it's provided.\"\n ([major minor f]\n (with-min major minor f nil))\n ([major minor f fallback-f]\n (if (is-at-least? major minor)\n (f)\n (when (not (nil? fallback-f)) (fallback-f)))))\n" }, { "path": "test/metabase/driver/clickhouse_data_types_test.clj", "content": "(ns metabase.driver.clickhouse-data-types-test\n #_{:clj-kondo/ignore [:unsorted-required-namespaces]}\n (:require [cljc.java-time.local-date :as local-date]\n [cljc.java-time.local-date-time :as local-date-time]\n [clojure.test :refer :all]\n [metabase.query-processor.test-util :as qp.test]\n [metabase.test :as mt]\n [metabase.test.data :as data]\n [metabase.test.data.clickhouse :as ctd]\n [metabase.test.data.interface :as tx]))\n\n(use-fixtures :once ctd/create-test-db!)\n\n(deftest ^:parallel clickhouse-decimals\n (mt/test-driver\n :clickhouse\n (data/dataset\n (tx/dataset-definition \"mbt\"\n [\"decimals\"\n [{:field-name \"my_money\"\n :base-type {:native \"Decimal(12,4)\"}}]\n [[1.0] [23.1337] [42.0] [42.0]]])\n (testing \"simple division\"\n (is\n (= 21.0\n (-> (data/run-mbql-query decimals\n {:expressions {:divided [:/ $my_money 2]}\n :filter [:> [:expression :divided] 1.0]\n :breakout [[:expression :divided]]\n :order-by [[:desc [:expression :divided]]]\n :limit 1})\n qp.test/first-row last float))))\n (testing \"divided decimal precision\"\n (is\n (= 1.8155331831916208\n (-> (data/run-mbql-query decimals\n {:expressions {:divided [:/ 42 $my_money]}\n :filter [:= $id 2]\n :limit 1})\n qp.test/first-row last double)))))))\n\n#_(deftest ^:parallel clickhouse-array-string\n (mt/test-driver\n :clickhouse\n (is\n (= \"[foo, bar]\"\n (-> (data/dataset\n (tx/dataset-definition \"metabase_tests_array_string\"\n [\"test-data-array-string\"\n [{:field-name \"my_array\"\n :base-type {:native \"Array(String)\"}}]\n [[(into-array (list \"foo\" \"bar\"))]]])\n (data/run-mbql-query test-data-array-string {:limit 1}))\n qp.test/first-row\n last)))))\n\n#_(deftest ^:parallel clickhouse-array-uint64\n (mt/test-driver\n :clickhouse\n (is\n (= \"[23, 42]\"\n (-> (data/dataset\n (tx/dataset-definition \"metabase_tests_array_uint\"\n [\"test-data-array-uint64\"\n [{:field-name \"my_array\"\n :base-type {:native \"Array(UInt64)\"}}]\n [[(into-array (list 23 42))]]])\n (data/run-mbql-query test-data-array-uint64 {:limit 1}))\n qp.test/first-row\n last)))))\n\n#_(deftest ^:parallel clickhouse-array-of-arrays\n (mt/test-driver\n :clickhouse\n (let [row1 (into-array (list\n (into-array (list \"foo\" \"bar\"))\n (into-array (list \"qaz\" \"qux\"))))\n row2 (into-array nil)\n query-result (data/dataset\n (tx/dataset-definition \"metabase_tests_array_of_arrays\"\n [\"test-data-array-of-arrays\"\n [{:field-name \"my_array_of_arrays\"\n :base-type {:native \"Array(Array(String))\"}}]\n [[row1] [row2]]])\n (data/run-mbql-query test-data-array-of-arrays {}))\n result (ctd/rows-without-index query-result)]\n (is (= [[\"[[foo, bar], [qaz, qux]]\"], [\"[]\"]] result)))))\n\n#_(deftest ^:parallel clickhouse-low-cardinality-array\n (mt/test-driver\n :clickhouse\n (let [row1 (into-array (list \"foo\" \"bar\"))\n row2 (into-array nil)\n query-result (data/dataset\n (tx/dataset-definition \"metabase_tests_low_cardinality_array\"\n [\"test-data-low-cardinality-array\"\n [{:field-name \"my_low_card_array\"\n :base-type {:native \"Array(LowCardinality(String))\"}}]\n [[row1] [row2]]])\n (data/run-mbql-query test-data-low-cardinality-array {}))\n result (ctd/rows-without-index query-result)]\n (is (= [[\"[foo, bar]\"], [\"[]\"]] result)))))\n\n#_(deftest ^:parallel clickhouse-array-of-nullables\n (mt/test-driver\n :clickhouse\n (let [row1 (into-array (list \"foo\" nil \"bar\"))\n row2 (into-array nil)\n query-result (data/dataset\n (tx/dataset-definition \"metabase_tests_array_of_nullables\"\n [\"test-data-array-of-nullables\"\n [{:field-name \"my_array_of_nullables\"\n :base-type {:native \"Array(Nullable(String))\"}}]\n [[row1] [row2]]])\n (data/run-mbql-query test-data-array-of-nullables {}))\n result (ctd/rows-without-index query-result)]\n (is (= [[\"[foo, null, bar]\"], [\"[]\"]] result)))))\n\n#_(deftest ^:parallel clickhouse-array-of-booleans\n (mt/test-driver\n :clickhouse\n (let [row1 (into-array (list true false true))\n row2 (into-array nil)\n query-result (data/dataset\n (tx/dataset-definition \"metabase_tests_array_of_booleans\"\n [\"test-data-array-of-booleans\"\n [{:field-name \"my_array_of_booleans\"\n :base-type {:native \"Array(Boolean)\"}}]\n [[row1] [row2]]])\n (data/run-mbql-query test-data-array-of-booleans {}))\n result (ctd/rows-without-index query-result)]\n (is (= [[\"[true, false, true]\"], [\"[]\"]] result)))))\n\n#_(deftest ^:parallel clickhouse-array-of-nullable-booleans\n (mt/test-driver\n :clickhouse\n (let [row1 (into-array (list true false nil))\n row2 (into-array nil)\n query-result (data/dataset\n (tx/dataset-definition \"metabase_tests_array_of_nullable_booleans\"\n [\"test-data-array-of-booleans\"\n [{:field-name \"my_array_of_nullable_booleans\"\n :base-type {:native \"Array(Nullable(Boolean))\"}}]\n [[row1] [row2]]])\n (data/run-mbql-query test-data-array-of-booleans {}))\n result (ctd/rows-without-index query-result)]\n (is (= [[\"[true, false, null]\"], [\"[]\"]] result)))))\n\n#_(deftest ^:parallel clickhouse-array-of-uint8\n (mt/test-driver\n :clickhouse\n (let [row1 (into-array (list 42 100 2))\n row2 (into-array nil)\n query-result (data/dataset\n (tx/dataset-definition \"metabase_tests_array_of_uint8\"\n [\"test-data-array-of-uint8\"\n [{:field-name \"my_array_of_uint8\"\n :base-type {:native \"Array(UInt8)\"}}]\n [[row1] [row2]]])\n (data/run-mbql-query test-data-array-of-uint8 {}))\n result (ctd/rows-without-index query-result)]\n (is (= [[\"[42, 100, 2]\"], [\"[]\"]] result)))))\n\n#_(deftest ^:parallel clickhouse-array-of-floats\n (mt/test-driver\n :clickhouse\n (let [row1 (into-array (list 1.2 3.4))\n row2 (into-array nil)\n query-result (data/dataset\n (tx/dataset-definition \"metabase_tests_array_of_floats\"\n [\"test-data-array-of-floats\"\n [{:field-name \"my_array_of_floats\"\n :base-type {:native \"Array(Float64)\"}}]\n [[row1] [row2]]])\n (data/run-mbql-query test-data-array-of-floats {}))\n result (ctd/rows-without-index query-result)]\n (is (= [[\"[1.2, 3.4]\"], [\"[]\"]] result)))))\n\n;; NB: timezones in the formatted string are purely cosmetic; it will be fine on the UI\n#_(deftest ^:parallel clickhouse-array-of-dates\n (mt/test-driver\n :clickhouse\n (let [row1 (into-array\n (list\n (local-date/parse \"2022-12-06\")\n (local-date/parse \"2021-10-19\")))\n row2 (into-array nil)\n query-result (data/dataset\n (tx/dataset-definition \"metabase_tests_array_of_dates\"\n [\"test-data-array-of-dates\"\n [{:field-name \"my_array_of_dates\"\n :base-type {:native \"Array(Date)\"}}]\n [[row1] [row2]]])\n (data/run-mbql-query test-data-array-of-dates {}))\n result (ctd/rows-without-index query-result)]\n (is (= [[\"[2022-12-06T00:00Z[UTC], 2021-10-19T00:00Z[UTC]]\"], [\"[]\"]] result)))))\n\n#_(deftest ^:parallel clickhouse-array-of-date32\n (mt/test-driver\n :clickhouse\n (let [row1 (into-array\n (list\n (local-date/parse \"2122-12-06\")\n (local-date/parse \"2099-10-19\")))\n row2 (into-array nil)\n query-result (data/dataset\n (tx/dataset-definition \"metabase_tests_array_of_date32\"\n [\"test-data-array-of-date32\"\n [{:field-name \"my_array_of_date32\"\n :base-type {:native \"Array(Date32)\"}}]\n [[row1] [row2]]])\n (data/run-mbql-query test-data-array-of-date32 {}))\n result (ctd/rows-without-index query-result)]\n (is (= [[\"[2122-12-06T00:00Z[UTC], 2099-10-19T00:00Z[UTC]]\"], [\"[]\"]] result)))))\n\n#_(deftest ^:parallel clickhouse-array-of-datetime\n (mt/test-driver\n :clickhouse\n (let [row1 (into-array\n (list\n (local-date-time/parse \"2022-12-06T18:28:31\")\n (local-date-time/parse \"2021-10-19T13:12:44\")))\n row2 (into-array nil)\n query-result (data/dataset\n (tx/dataset-definition \"metabase_tests_array_of_datetime\"\n [\"test-data-array-of-datetime\"\n [{:field-name \"my_array_of_datetime\"\n :base-type {:native \"Array(DateTime)\"}}]\n [[row1] [row2]]])\n (data/run-mbql-query test-data-array-of-datetime {}))\n result (ctd/rows-without-index query-result)]\n (is (= [[\"[2022-12-06T18:28:31Z[UTC], 2021-10-19T13:12:44Z[UTC]]\"], [\"[]\"]] result)))))\n\n#_(deftest ^:parallel clickhouse-array-of-datetime64\n (mt/test-driver\n :clickhouse\n (let [row1 (into-array\n (list\n (local-date-time/parse \"2022-12-06T18:28:31.123\")\n (local-date-time/parse \"2021-10-19T13:12:44.456\")))\n row2 (into-array nil)\n query-result (data/dataset\n (tx/dataset-definition \"metabase_tests_array_of_datetime64\"\n [\"test-data-array-of-datetime64\"\n [{:field-name \"my_array_of_datetime64\"\n :base-type {:native \"Array(DateTime64(3))\"}}]\n [[row1] [row2]]])\n (data/run-mbql-query test-data-array-of-datetime64 {}))\n result (ctd/rows-without-index query-result)]\n (is (= [[\"[2022-12-06T18:28:31.123Z[UTC], 2021-10-19T13:12:44.456Z[UTC]]\"], [\"[]\"]] result)))))\n\n#_(deftest ^:parallel clickhouse-array-of-decimals\n (mt/test-driver\n :clickhouse\n (let [row1 (into-array (list \"12345123.123456789\" \"78.245\"))\n row2 nil\n query-result (data/dataset\n (tx/dataset-definition \"metabase_tests_array_of_decimals\"\n [\"test-data-array-of-decimals\"\n [{:field-name \"my_array_of_decimals\"\n :base-type {:native \"Array(Decimal(18, 9))\"}}]\n [[row1] [row2]]])\n (data/run-mbql-query test-data-array-of-decimals {}))\n result (ctd/rows-without-index query-result)]\n (is (= [[\"[12345123.123456789, 78.245000000]\"], [\"[]\"]] result)))))\n\n#_(deftest ^:parallel clickhouse-array-of-tuples\n (mt/test-driver\n :clickhouse\n (is (= [[\"[[foobar, 1234], [qaz, 0]]\"]\n [\"[]\"]]\n (qp.test/formatted-rows\n [str]\n :format-nil-values\n (ctd/do-with-test-db\n (fn [db]\n (data/with-db db\n (data/run-mbql-query\n array_of_tuples_test\n {})))))))))\n\n#_(deftest ^:parallel clickhouse-array-of-uuids\n (mt/test-driver\n :clickhouse\n (let [row1 (into-array (list \"2eac427e-7596-11ed-a1eb-0242ac120002\"\n \"2eac44f4-7596-11ed-a1eb-0242ac120002\"))\n row2 nil\n query-result (data/dataset\n (tx/dataset-definition \"metabase_tests_array_of_uuids\"\n [\"test-data-array-of-uuids\"\n [{:field-name \"my_array_of_uuids\"\n :base-type {:native \"Array(UUID)\"}}]\n [[row1] [row2]]])\n (data/run-mbql-query test-data-array-of-uuids {}))\n result (ctd/rows-without-index query-result)]\n (is (= [[\"[2eac427e-7596-11ed-a1eb-0242ac120002, 2eac44f4-7596-11ed-a1eb-0242ac120002]\"], [\"[]\"]] result)))))\n\n#_(deftest ^:parallel clickhouse-array-inner-types\n (mt/test-driver\n :clickhouse\n (is (= [[\"[a, b, c]\"\n \"[null, d, e]\"\n \"[1.0000, 2.0000, 3.0000]\"\n \"[4.0000, null, 5.0000]\"]]\n (ctd/do-with-test-db\n (fn [db]\n (data/with-db db\n (->> (data/run-mbql-query arrays_inner_types {})\n (mt/formatted-rows [str str str str])))))))))\n\n(deftest ^:parallel clickhouse-nullable-strings\n (mt/test-driver\n :clickhouse\n (data/dataset\n (tx/dataset-definition\n \"metabase_tests_nullable_strings\"\n [\"test-data-nullable-strings\"\n [{:field-name \"mystring\" :base-type :type/Text}]\n [[\"foo\"] [\"bar\"] [\" \"] [\"\"] [nil]]])\n (testing \"null strings count\"\n (is (= 2M ;; BigDecimal\n (-> (data/run-mbql-query test-data-nullable-strings\n {:filter [:is-null $mystring]\n :aggregation [:count]})\n qp.test/first-row last))))\n (testing \"nullable strings not null filter\"\n (is (= 3M\n (-> (data/run-mbql-query test-data-nullable-strings\n {:filter [:not-null $mystring]\n :aggregation [:count]})\n qp.test/first-row last))))\n (testing \"filter nullable string by value\"\n (is (= 1M\n (-> (data/run-mbql-query test-data-nullable-strings\n {:filter [:= $mystring \"foo\"]\n :aggregation [:count]})\n qp.test/first-row last)))))))\n\n(deftest ^:parallel clickhouse-non-latin-strings\n (mt/test-driver\n :clickhouse\n (testing \"basic filtering\"\n (is (= [[1 \"Я_1\"] [3 \"Я_2\"] [4 \"Я\"]]\n (qp.test/formatted-rows\n [int str]\n :format-nil-values\n (ctd/do-with-test-db\n (fn [db]\n (data/with-db db\n (data/run-mbql-query\n metabase_test_lowercases\n {:filter [:contains $mystring \"Я\"]}))))))))\n (testing \"case-insensitive non-latin filtering\"\n (is (= [[1 \"Я_1\"] [3 \"Я_2\"] [4 \"Я\"] [5 \"я\"]]\n (qp.test/formatted-rows\n [int str]\n :format-nil-values\n (ctd/do-with-test-db\n (fn [db]\n (data/with-db db\n (data/run-mbql-query\n metabase_test_lowercases\n {:filter [:contains $mystring \"Я\"\n {:case-sensitive false}]}))))))))))\n\n(deftest ^:parallel clickhouse-datetime64-filter\n (mt/test-driver\n :clickhouse\n (let [row1 \"2022-03-03 03:03:03.333\"\n row2 \"2022-03-03 03:03:03.444\"\n row3 \"2022-03-03 03:03:03\"\n query-result (data/dataset\n (tx/dataset-definition \"metabase_tests_datetime64\"\n [\"test-data-datetime64\"\n [{:field-name \"milli_sec\"\n :base-type {:native \"DateTime64(3)\"}}]\n [[row1] [row2] [row3]]])\n (data/run-mbql-query test-data-datetime64 {:filter [:= $milli_sec \"2022-03-03T03:03:03.333Z\"]}))\n result (ctd/rows-without-index query-result)]\n (is (= [[\"2022-03-03T03:03:03.333Z\"]] result)))))\n\n(deftest ^:parallel clickhouse-datetime-filter\n (mt/test-driver\n :clickhouse\n (let [row1 \"2022-03-03 03:03:03\"\n row2 \"2022-03-03 03:03:04\"\n row3 \"2022-03-03 03:03:05\"\n query-result (data/dataset\n (tx/dataset-definition \"metabase_tests_datetime\"\n [\"test-data-datetime\"\n [{:field-name \"second\"\n :base-type {:native \"DateTime\"}}]\n [[row1] [row2] [row3]]])\n (data/run-mbql-query test-data-datetime {:filter [:= $second \"2022-03-03T03:03:04Z\"]}))\n result (ctd/rows-without-index query-result)]\n (is (= [[\"2022-03-03T03:03:04Z\"]] result)))))\n\n(deftest ^:parallel clickhouse-booleans\n (mt/test-driver\n :clickhouse\n (let [[row1 row2 row3 row4] [[\"#1\" true] [\"#2\" false] [\"#3\" false] [\"#4\" true]]\n query-result (data/dataset\n (tx/dataset-definition \"metabase_tests_booleans\"\n [\"test-data-booleans\"\n [{:field-name \"name\"\n :base-type :type/Text}\n {:field-name \"is_active\"\n :base-type :type/Boolean}]\n [row1 row2 row3 row4]])\n (data/run-mbql-query test-data-booleans {:filter [:= $is_active false]}))\n rows (qp.test/rows query-result)\n result (map #(drop 1 %) rows)] ; remove db \"index\" which is the first column in the result set\n (is (= [row2 row3] result)))))\n\n(deftest ^:parallel clickhouse-enums-values-test\n (mt/test-driver\n :clickhouse\n (testing \"select enums values as strings\"\n (is (= [[\"foo\" \"house\" \"qaz\"]\n [\"foo bar\" \"click\" \"qux\"]\n [\"bar\" \"house\" \"qaz\"]]\n (qp.test/formatted-rows\n [str str str]\n :format-nil-values\n (ctd/do-with-test-db\n (fn [db]\n (data/with-db db\n (data/run-mbql-query\n enums_test\n {}))))))))\n (testing \"filtering enum values\"\n (is (= [[\"useqa\"]]\n (qp.test/formatted-rows\n [str]\n :format-nil-values\n (ctd/do-with-test-db\n (fn [db]\n (data/with-db db\n (data/run-mbql-query\n enums_test\n {:expressions {\"test\" [:concat\n [:substring $enum2 3 3]\n [:substring $enum3 1 2]]}\n :fields [[:expression \"test\"]]\n :filter [:= $enum1 \"foo\"]}))))))))))\n\n(deftest ^:parallel clickhouse-ipv4query-test\n (mt/test-driver\n :clickhouse\n (is (= [[1]]\n (qp.test/formatted-rows\n [int]\n :format-nil-values\n (ctd/do-with-test-db\n (fn [db]\n (data/with-db db\n (data/run-mbql-query\n ipaddress_test\n {:filter [:= $ipvfour \"127.0.0.1\"]\n :aggregation [[:count]]})))))))))\n\n(deftest ^:parallel clickhouse-ip-serialization-test\n (mt/test-driver\n :clickhouse\n (is (= [[\"127.0.0.1\" \"0:0:0:0:0:0:0:1\"]\n [\"0.0.0.0\" \"2001:438:ffff:0:0:0:407d:1bc1\"]\n [nil nil]]\n (qp.test/formatted-rows\n [str str]\n (ctd/do-with-test-db\n (fn [db] (data/with-db db (data/run-mbql-query ipaddress_test {})))))))))\n\n(defn- map-as-string [^java.util.LinkedHashMap m] (.toString m))\n(deftest ^:parallel clickhouse-simple-map-test\n (mt/test-driver\n :clickhouse\n (is (= [[\"{key1=1, key2=10}\"] [\"{key1=2, key2=20}\"] [\"{key1=3, key2=30}\"]]\n (qp.test/formatted-rows\n [map-as-string]\n :format-nil-values\n (ctd/do-with-test-db\n (fn [db]\n (data/with-db db\n (data/run-mbql-query\n maps_test\n {})))))))))\n\n(deftest ^:parallel clickhouse-datetime-diff-nullable\n (mt/test-driver\n :clickhouse\n (is (= [[170 202] [nil nil] [nil nil] [nil nil]]\n (ctd/do-with-test-db\n (fn [db]\n (data/with-db db\n (->> (data/run-mbql-query\n datetime_diff_nullable\n {:fields [[:expression \"dt64,dt\"]\n [:expression \"dt64,d\"]]\n :expressions\n {\"dt64,dt\" [:datetime-diff $dt64 $dt :day]\n \"dt64,d\" [:datetime-diff $dt64 $d :day]}})\n (mt/formatted-rows [int int])))))))))\n\n;; Metabase has pretty extensive testing for sum-where and count-where\n;; However, this ClickHouse-specific corner case is not covered\n(deftest ^:parallel clickhouse-sum-where-numeric-types\n (mt/test-driver\n :clickhouse\n (testing \"int values (with matching rows)\"\n (is (= [[8]]\n (qp.test/formatted-rows\n [int]\n :format-nil-values\n (ctd/do-with-test-db\n (fn [db]\n (data/with-db db\n (data/run-mbql-query\n sum_if_test_int\n {:aggregation [[:sum-where $int_value [:= $discriminator \"bar\"]]]}))))))))\n (testing \"int values (no matching rows)\"\n (is (= [[0]]\n (qp.test/formatted-rows\n [int]\n :format-nil-values\n (ctd/do-with-test-db\n (fn [db]\n (data/with-db db\n (data/run-mbql-query\n sum_if_test_int\n {:aggregation [[:sum-where $int_value [:= $discriminator \"qaz\"]]]}))))))))\n (testing \"double values (with matching rows)\"\n (is (= [[9.27]]\n (qp.test/formatted-rows\n [double]\n :format-nil-values\n (ctd/do-with-test-db\n (fn [db]\n (data/with-db db\n (data/run-mbql-query\n sum_if_test_float\n {:aggregation [[:sum-where $float_value [:= $discriminator \"bar\"]]]}))))))))\n (testing \"double values (no matching rows)\"\n (is (= [[0.0]]\n (qp.test/formatted-rows\n [double]\n :format-nil-values\n (ctd/do-with-test-db\n (fn [db]\n (data/with-db db\n (data/run-mbql-query\n sum_if_test_float\n {:aggregation [[:sum-where $float_value [:= $discriminator \"qaz\"]]]}))))))))))\n\n(deftest ^:parallel clickhouse-unsigned-integers\n (mt/test-driver\n :clickhouse\n (is (= [[\"255\" \"65535\" \"4294967295\" \"18446744073709551615\"]]\n (qp.test/formatted-rows\n [str str str str]\n :format-nil-values\n (ctd/do-with-test-db\n (fn [db]\n (data/with-db db\n (data/run-mbql-query\n unsigned_int_types\n {})))))))))\n\n;; FIXME: blocked by https://github.com/ClickHouse/clickhouse-java/issues/2218\n#_(deftest ^:parallel clickhouse-fixed-strings\n (mt/test-driver\n :clickhouse\n (is (= [[\"val1\" \"val2\" \"val3\" \"val4\"]]\n (qp.test/formatted-rows\n [str str str str]\n :format-nil-values\n (ctd/do-with-test-db\n (fn [db]\n (data/with-db db\n (data/run-mbql-query\n fixed_strings\n {})))))))))\n" }, { "path": "test/metabase/driver/clickhouse_impersonation_test.clj", "content": "(ns metabase.driver.clickhouse-impersonation-test\n \"SET ROLE (connection impersonation feature) tests on with single node or on-premise cluster setups.\"\n #_{:clj-kondo/ignore [:unsorted-required-namespaces]}\n (:require [clojure.test :refer :all]\n [metabase-enterprise.advanced-permissions.api.util-test :as advanced-perms.api.tu]\n [metabase.driver :as driver]\n [metabase.driver.sql :as driver.sql]\n [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]\n [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]\n [metabase.query-processor.store :as qp.store]\n [metabase.test :as mt]\n [metabase.test.data.clickhouse :as ctd]\n [metabase.util :as u]\n [toucan2.tools.with-temp :as t2.with-temp]))\n\n;; 53+ metabase.sync has moved to metabase.sync.core\n(try\n (require '[metabase.sync :as sync])\n (catch java.io.FileNotFoundException e\n (when (re-find #\"metabase/sync\\.clj\" (.getMessage e))\n (require '[metabase.sync.core :as sync]))))\n\n(set! *warn-on-reflection* true)\n\n(defn- set-role-test!\n [details-map]\n (let [default-role (driver.sql/default-database-role :clickhouse nil)\n spec (sql-jdbc.conn/connection-details->spec :clickhouse details-map)]\n (testing \"default role is NONE\"\n (is (= default-role \"NONE\")))\n (testing \"does not throw with an existing role\"\n (sql-jdbc.execute/do-with-connection-with-options\n :clickhouse spec nil\n (fn [^java.sql.Connection conn]\n (driver/set-role! :clickhouse conn \"metabase_test_role\")))\n (is true))\n (testing \"does not throw with a role containing hyphens\"\n (sql-jdbc.execute/do-with-connection-with-options\n :clickhouse spec nil\n (fn [^java.sql.Connection conn]\n (driver/set-role! :clickhouse conn \"metabase-test-role\")))\n (is true))\n (testing \"does not throw with the default role\"\n (sql-jdbc.execute/do-with-connection-with-options\n :clickhouse spec nil\n (fn [^java.sql.Connection conn]\n (driver/set-role! :clickhouse conn default-role)\n (fn [^java.sql.Connection conn]\n (driver/set-role! :clickhouse conn default-role)\n (with-open [stmt (.prepareStatement conn \"SELECT * FROM `metabase_test_role_db`.`some_table` ORDER BY i ASC;\")\n rset (.executeQuery stmt)]\n (is (.next rset) true)\n (is (.getInt rset 1) 42)\n (is (.next rset) true)\n (is (.getInt rset 1) 144)\n (is (.next rset) false)))))\n (is true))))\n\n(defn- set-role-throws-test!\n [details-map]\n (testing \"throws when assigning a non-existent role\"\n (is (thrown? Exception\n (sql-jdbc.execute/do-with-connection-with-options\n :clickhouse (sql-jdbc.conn/connection-details->spec :clickhouse details-map) nil\n (fn [^java.sql.Connection conn]\n (driver/set-role! :clickhouse conn \"asdf\")))))))\n\n(defn- do-with-new-metadata-provider\n [details thunk]\n (t2.with-temp/with-temp\n [:model/Database db {:engine :clickhouse :details details}]\n (qp.store/with-metadata-provider (u/the-id db) (thunk db))))\n\n(deftest clickhouse-set-role\n (mt/test-driver\n :clickhouse\n (let [user-details {:user \"metabase_test_user\"}\n ;; See docker-compose.yml for the port mappings\n ;; 24.4+\n single-node-port-details {:port 8123}\n single-node-details (merge user-details single-node-port-details)\n cluster-port-details {:port 8127}\n cluster-details (merge user-details cluster-port-details)]\n (testing \"single node\"\n (testing \"should support the impersonation feature\"\n (t2.with-temp/with-temp\n [:model/Database db {:engine :clickhouse :details {:user \"default\" :port 8123}}]\n (is (true? (driver/database-supports? :clickhouse :connection-impersonation db)))))\n (let [statements [\"CREATE DATABASE IF NOT EXISTS `metabase_test_role_db`;\"\n \"CREATE OR REPLACE TABLE `metabase_test_role_db`.`some_table` (i Int32) ENGINE = MergeTree ORDER BY (i);\"\n \"INSERT INTO `metabase_test_role_db`.`some_table` VALUES (42), (144);\"\n \"CREATE ROLE IF NOT EXISTS `metabase_test_role`;\"\n \"CREATE ROLE IF NOT EXISTS `metabase-test-role`;\"\n \"CREATE USER IF NOT EXISTS `metabase_test_user` NOT IDENTIFIED;\"\n \"GRANT SELECT ON `metabase_test_role_db`.* TO `metabase_test_role`,`metabase-test-role`;\"\n \"GRANT `metabase_test_role`, `metabase-test-role` TO `metabase_test_user`;\"]]\n (ctd/exec-statements statements single-node-port-details)\n (do-with-new-metadata-provider\n single-node-details\n (fn [_db]\n (set-role-test! single-node-details)\n (set-role-throws-test! single-node-details)))))\n (testing \"on-premise cluster\"\n (testing \"should support the impersonation feature\"\n (t2.with-temp/with-temp\n [:model/Database db {:engine :clickhouse :details {:user \"default\" :port 8127}}]\n (is (true? (driver/database-supports? :clickhouse :connection-impersonation db)))))\n (let [statements [\"CREATE DATABASE IF NOT EXISTS `metabase_test_role_db` ON CLUSTER '{cluster}';\"\n \"CREATE OR REPLACE TABLE `metabase_test_role_db`.`some_table` ON CLUSTER '{cluster}' (i Int32)\n ENGINE ReplicatedMergeTree('/clickhouse/{cluster}/tables/{database}/{table}/{shard}', '{replica}')\n ORDER BY (i);\"\n \"INSERT INTO `metabase_test_role_db`.`some_table` VALUES (42), (144);\"\n \"CREATE ROLE IF NOT EXISTS `metabase_test_role` ON CLUSTER '{cluster}';\"\n \"CREATE ROLE IF NOT EXISTS `metabase-test-role` ON CLUSTER '{cluster}';\"\n \"CREATE USER IF NOT EXISTS `metabase_test_user` ON CLUSTER '{cluster}' NOT IDENTIFIED;\"\n \"GRANT ON CLUSTER '{cluster}' SELECT ON `metabase_test_role_db`.* TO `metabase_test_role`, `metabase-test-role`;\"\n \"GRANT ON CLUSTER '{cluster}' `metabase_test_role`, `metabase-test-role` TO `metabase_test_user`;\"]]\n (ctd/exec-statements statements cluster-port-details)\n (do-with-new-metadata-provider\n cluster-details\n (fn [_db]\n (set-role-test! cluster-details)\n (set-role-throws-test! cluster-details)))))\n (testing \"older ClickHouse version\" ;; 23.3\n (testing \"should NOT support the impersonation feature\"\n (t2.with-temp/with-temp\n [:model/Database db {:engine :clickhouse :details {:user \"default\" :port 8124}}]\n (is (false? (driver/database-supports? :clickhouse :connection-impersonation db)))))))))\n\n(deftest conn-impersonation-test-clickhouse\n (mt/test-driver\n :clickhouse\n (mt/with-premium-features #{:advanced-permissions}\n (let [table-name (str \"metabase_impersonation_test.test_\" (System/currentTimeMillis))\n select-query (format \"SELECT * FROM %s;\" table-name)\n cluster-port {:port 8127}\n cluster-details {:engine :clickhouse\n :details {:user \"metabase_impersonation_test_user\"\n :dbname \"metabase_impersonation_test\"\n :port 8127}}\n ddl-statements [\"CREATE DATABASE IF NOT EXISTS metabase_impersonation_test ON CLUSTER '{cluster}';\"\n (format \"CREATE TABLE %s ON CLUSTER '{cluster}' (s String)\n ENGINE ReplicatedMergeTree('/clickhouse/{cluster}/tables/{database}/{table}/{shard}', '{replica}')\n ORDER BY (s);\" table-name)]\n insert-statements [(format \"INSERT INTO %s VALUES ('a'), ('b'), ('c');\" table-name)]\n grant-statements [\"CREATE USER IF NOT EXISTS metabase_impersonation_test_user ON CLUSTER '{cluster}' NOT IDENTIFIED;\"\n \"CREATE ROLE IF NOT EXISTS row_a ON CLUSTER '{cluster}';\"\n \"CREATE ROLE IF NOT EXISTS row_b ON CLUSTER '{cluster}';\"\n \"CREATE ROLE IF NOT EXISTS row_c ON CLUSTER '{cluster}';\"\n \"GRANT ON CLUSTER '{cluster}' row_a, row_b, row_c TO metabase_impersonation_test_user;\"\n (format \"GRANT ON CLUSTER '{cluster}' SELECT ON %s TO metabase_impersonation_test_user;\" table-name)\n (format \"CREATE ROW POLICY OR REPLACE policy_row_a ON CLUSTER '{cluster}'\n ON %s FOR SELECT USING s = 'a' TO row_a;\" table-name)\n (format \"CREATE ROW POLICY OR REPLACE policy_row_b ON CLUSTER '{cluster}'\n ON %s FOR SELECT USING s = 'b' TO row_b;\" table-name)\n (format \"CREATE ROW POLICY OR REPLACE policy_row_c ON CLUSTER '{cluster}'\n ON %s FOR SELECT USING s = 'c' TO row_c;\" table-name)]]\n (ctd/exec-statements ddl-statements cluster-port {\"wait_end_of_query\" \"1\"})\n (ctd/exec-statements insert-statements cluster-port {\"wait_end_of_query\" \"1\"\n \"insert_quorum\" \"2\"})\n (ctd/exec-statements grant-statements cluster-port {\"wait_end_of_query\" \"1\"})\n (t2.with-temp/with-temp [:model/Database db cluster-details]\n (mt/with-db db (sync/sync-database! db)\n\n (letfn [(check-impersonation! [roles expected]\n (advanced-perms.api.tu/with-impersonations!\n {:impersonations [{:db-id (mt/id) :attribute \"impersonation_attr\"}]\n :attributes {\"impersonation_attr\" roles}}\n (is (= expected\n (-> {:query select-query}\n mt/native-query\n mt/process-query\n mt/rows)))))]\n\n (is (= [[\"a\"] [\"b\"] [\"c\"]]\n (-> {:query select-query}\n mt/native-query\n mt/process-query\n mt/rows)))\n\n (check-impersonation! \"row_a\" [[\"a\"]])\n (check-impersonation! \"row_b\" [[\"b\"]])\n (check-impersonation! \"row_c\" [[\"c\"]])\n (check-impersonation! \"row_a,row_c\" [[\"a\"] [\"c\"]])\n (check-impersonation! \"row_b,row_c\" [[\"b\"] [\"c\"]])\n (check-impersonation! \"row_a,row_b,row_c\" [[\"a\"] [\"b\"] [\"c\"]]))))))))\n" }, { "path": "test/metabase/driver/clickhouse_introspection_test.clj", "content": "(ns metabase.driver.clickhouse-introspection-test\n #_{:clj-kondo/ignore [:unsorted-required-namespaces]}\n (:require\n [clojure.test :refer :all]\n [metabase.driver :as driver]\n [metabase.driver.common :as driver.common]\n [metabase.query-processor :as qp]\n [metabase.query-processor.test-util :as qp.test]\n [metabase.test :as mt]\n [metabase.test.data :as data]\n [metabase.test.data.clickhouse :as ctd]\n [metabase.test.data.interface :as tx]\n [toucan2.tools.with-temp :as t2.with-temp]))\n\n(use-fixtures :once ctd/create-test-db!)\n\n(defn- desc-table\n [table-name]\n (into #{} (map #(select-keys % [:name :database-type :base-type :database-required])\n (:fields (ctd/do-with-test-db\n #(driver/describe-table :clickhouse % {:name table-name}))))))\n\n(deftest ^:parallel clickhouse-base-types-test-enums\n (mt/test-driver\n :clickhouse\n (testing \"enums\"\n (let [table-name \"enums_base_types\"]\n (is (= #{{:base-type :type/Text,\n :database-required false,\n :database-type \"Nullable(Enum8('America/New_York' = 1))\",\n :name \"c1\"}\n {:base-type :type/Text,\n :database-required true,\n :database-type \"Enum8('BASE TABLE' = 1, 'VIEW' = 2, 'FOREIGN TABLE' = 3, 'LOCAL TEMPORARY' = 4, 'SYSTEM VIEW' = 5)\",\n :name \"c2\"}\n {:base-type :type/Text,\n :database-required true,\n :database-type \"Enum8('NO' = 1, 'YES' = 2)\",\n :name \"c3\"}\n {:base-type :type/Text,\n :database-required true,\n :database-type \"Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2)\",\n :name \"c4\"}\n {:base-type :type/Text,\n :database-required false,\n :database-type \"Nullable(Enum8('GLOBAL' = 0, 'DATABASE' = 1, 'TABLE' = 2))\",\n :name \"c5\"}\n {:base-type :type/Text,\n :database-required false,\n :database-type \"Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2))\",\n :name \"c6\"}}\n (desc-table table-name)))))))\n\n(deftest ^:parallel clickhouse-base-types-test-dates\n (mt/test-driver\n :clickhouse\n (testing \"dates\"\n (let [table-name \"date_base_types\"]\n (is (= #{{:base-type :type/Date,\n :database-required true,\n :database-type \"Date\",\n :name \"c1\"}\n {:base-type :type/Date,\n :database-required true,\n :database-type \"Date32\",\n :name \"c2\"}\n {:base-type :type/Date,\n :database-required false,\n :database-type \"Nullable(Date)\",\n :name \"c3\"}\n {:base-type :type/Date,\n :database-required false,\n :database-type \"Nullable(Date32)\",\n :name \"c4\"}}\n (desc-table table-name)))))))\n\n(deftest ^:parallel clickhouse-base-types-test-datetimes\n (mt/test-driver\n :clickhouse\n (testing \"datetimes\"\n (let [table-name \"datetime_base_types\"]\n (is (= #{{:base-type :type/DateTimeWithLocalTZ,\n :database-required false,\n :database-type \"Nullable(DateTime('America/New_York'))\",\n :name \"c1\"}\n {:base-type :type/DateTimeWithLocalTZ,\n :database-required true,\n :database-type \"DateTime('America/New_York')\",\n :name \"c2\"}\n {:base-type :type/DateTimeWithLocalTZ,\n :database-required true,\n :database-type \"DateTime\",\n :name \"c3\"}\n {:base-type :type/DateTimeWithLocalTZ,\n :database-required true,\n :database-type \"DateTime64(3)\",\n :name \"c4\"}\n {:base-type :type/DateTimeWithLocalTZ,\n :database-required true,\n :database-type \"DateTime64(9, 'America/New_York')\",\n :name \"c5\"}\n {:base-type :type/DateTimeWithLocalTZ,\n :database-required false,\n :database-type \"Nullable(DateTime64(6, 'America/New_York'))\",\n :name \"c6\"}\n {:base-type :type/DateTimeWithLocalTZ,\n :database-required false,\n :database-type \"Nullable(DateTime64(0))\",\n :name \"c7\"}\n {:base-type :type/DateTimeWithLocalTZ,\n :database-required false,\n :database-type \"Nullable(DateTime)\",\n :name \"c8\"}}\n (desc-table table-name)))))))\n\n(deftest ^:parallel clickhouse-base-types-test-integers\n (mt/test-driver\n :clickhouse\n (testing \"integers\"\n (let [table-name \"integer_base_types\"]\n (is (= #{{:base-type :type/Integer,\n :database-required true,\n :database-type \"UInt8\",\n :name \"c1\"}\n {:base-type :type/Integer,\n :database-required true,\n :database-type \"UInt16\",\n :name \"c2\"}\n {:base-type :type/Integer,\n :database-required true,\n :database-type \"UInt32\",\n :name \"c3\"}\n {:base-type :type/BigInteger,\n :database-required true,\n :database-type \"UInt64\",\n :name \"c4\"}\n {:base-type :type/*,\n :database-required true,\n :database-type \"UInt128\",\n :name \"c5\"}\n {:base-type :type/*,\n :database-required true,\n :database-type \"UInt256\",\n :name \"c6\"}\n {:base-type :type/Integer,\n :database-required true,\n :database-type \"Int8\",\n :name \"c7\"}\n {:base-type :type/Integer,\n :database-required true,\n :database-type \"Int16\",\n :name \"c8\"}\n {:base-type :type/Integer,\n :database-required true,\n :database-type \"Int32\",\n :name \"c9\"}\n {:base-type :type/BigInteger,\n :database-required true,\n :database-type \"Int64\",\n :name \"c10\"}\n {:base-type :type/*,\n :database-required true,\n :database-type \"Int128\",\n :name \"c11\"}\n {:base-type :type/*,\n :database-required true,\n :database-type \"Int256\",\n :name \"c12\"}\n {:base-type :type/Integer,\n :database-required false,\n :database-type \"Nullable(Int32)\",\n :name \"c13\"}}\n (desc-table table-name)))))))\n\n(deftest ^:parallel clickhouse-base-types-test-numerics\n (mt/test-driver\n :clickhouse\n (testing \"numerics\"\n (let [table-name \"numeric_base_types\"]\n (is (= #{{:base-type :type/Float,\n :database-required true,\n :database-type \"Float32\",\n :name \"c1\"}\n {:base-type :type/Float,\n :database-required true,\n :database-type \"Float64\",\n :name \"c2\"}\n {:base-type :type/Decimal,\n :database-required true,\n :database-type \"Decimal(4, 2)\",\n :name \"c3\"}\n {:base-type :type/Decimal,\n :database-required true,\n :database-type \"Decimal(9, 7)\",\n :name \"c4\"}\n {:base-type :type/Decimal,\n :database-required true,\n :database-type \"Decimal(18, 12)\",\n :name \"c5\"}\n {:base-type :type/Decimal,\n :database-required true,\n :database-type \"Decimal(38, 24)\",\n :name \"c6\"}\n {:base-type :type/Decimal,\n :database-required true,\n :database-type \"Decimal(76, 42)\",\n :name \"c7\"}\n {:base-type :type/Float,\n :database-required false,\n :database-type \"Nullable(Float32)\",\n :name \"c8\"}\n {:base-type :type/Decimal,\n :database-required false,\n :database-type \"Nullable(Decimal(4, 2))\",\n :name \"c9\"}\n {:base-type :type/Decimal,\n :database-required false,\n :database-type \"Nullable(Decimal(76, 42))\",\n :name \"c10\"}}\n (desc-table table-name)))))))\n\n(deftest ^:parallel clickhouse-base-types-test-strings\n (mt/test-driver\n :clickhouse\n (testing \"strings\"\n (let [table-name \"string_base_types\"]\n (is (= #{{:base-type :type/Text,\n :database-required true,\n :database-type \"String\",\n :name \"c1\"}\n {:base-type :type/Text,\n :database-required true,\n :database-type \"LowCardinality(String)\",\n :name \"c2\"}\n {:base-type :type/TextLike,\n :database-required true,\n :database-type \"FixedString(32)\",\n :name \"c3\"}\n {:base-type :type/Text,\n :database-required false,\n :database-type \"Nullable(String)\",\n :name \"c4\"}\n {:base-type :type/TextLike,\n :database-required true,\n :database-type \"LowCardinality(FixedString(4))\",\n :name \"c5\"}}\n (desc-table table-name)))))))\n\n(deftest ^:parallel clickhouse-base-types-test-arrays\n (mt/test-driver\n :clickhouse\n (testing \"arrays\"\n (let [table-name \"array_base_types\"]\n (is (= #{{:base-type :type/Array,\n :database-required true,\n :database-type \"Array(String)\",\n :name \"c1\"}\n {:base-type :type/Array,\n :database-required true,\n :database-type \"Array(Nullable(Int32))\",\n :name \"c2\"}\n {:base-type :type/Array,\n :database-required true,\n :database-type \"Array(Array(LowCardinality(FixedString(32))))\",\n :name \"c3\"}\n {:base-type :type/Array,\n :database-required true,\n :database-type \"Array(Array(Array(String)))\",\n :name \"c4\"}}\n (desc-table table-name)))))))\n\n(deftest ^:parallel clickhouse-base-types-test-low-cardinality-nullable\n (mt/test-driver\n :clickhouse\n (testing \"low cardinality nullable\"\n (let [table-name \"low_cardinality_nullable_base_types\"]\n (is (= #{{:base-type :type/Text,\n :database-required true,\n :database-type \"LowCardinality(Nullable(String))\",\n :name \"c1\"}\n {:base-type :type/TextLike,\n :database-required true,\n :database-type \"LowCardinality(Nullable(FixedString(16)))\",\n :name \"c2\"}}\n (desc-table table-name)))))))\n\n(deftest ^:parallel clickhouse-base-types-test-misc\n (mt/test-driver\n :clickhouse\n (testing \"everything else\"\n (let [table-name \"misc_base_types\"]\n (is (= #{{:base-type :type/Boolean,\n :database-required true,\n :database-type \"Bool\",\n :name \"c1\"}\n {:base-type :type/UUID,\n :database-required true,\n :database-type \"UUID\",\n :name \"c2\"}\n {:base-type :type/IPAddress,\n :database-required true,\n :database-type \"IPv4\",\n :name \"c3\"}\n {:base-type :type/IPAddress,\n :database-required true,\n :database-type \"IPv6\",\n :name \"c4\"}\n {:base-type :type/Dictionary,\n :database-required true,\n :database-type \"Map(Int32, String)\",\n :name \"c5\"}\n {:base-type :type/Boolean,\n :database-required false,\n :database-type \"Nullable(Bool)\",\n :name \"c6\"}\n {:base-type :type/UUID,\n :database-required false,\n :database-type \"Nullable(UUID)\",\n :name \"c7\"}\n {:base-type :type/IPAddress,\n :database-required false,\n :database-type \"Nullable(IPv4)\",\n :name \"c8\"}\n {:base-type :type/IPAddress,\n :database-required false,\n :database-type \"Nullable(IPv6)\",\n :name \"c9\"}\n {:base-type :type/*,\n :database-required true,\n :database-type \"Tuple(String, Int32)\",\n :name \"c10\"}}\n (desc-table table-name)))))))\n\n(deftest ^:parallel clickhouse-boolean-type-metadata\n (mt/test-driver\n :clickhouse\n (let [result (-> {:query \"SELECT false, 123, true\"} mt/native-query qp/process-query)\n [[c1 _ c3]] (-> result qp.test/rows)]\n (testing \"column should be of type :type/Boolean\"\n (is (= :type/Boolean (-> result :data :results_metadata :columns first :base_type)))\n (is (= :type/Boolean (transduce identity (driver.common/values->base-type) [c1, c3])))\n (is (= :type/Boolean (driver.common/class->base-type (class c1))))))))\n\n(def ^:private base-field\n {:database-is-auto-increment false\n :json-unfolding false\n :database-required true})\n\n(deftest ^:parallel clickhouse-filtered-aggregate-functions-test-table-metadata\n (mt/test-driver\n :clickhouse\n (is (= {:name \"aggregate_functions_filter_test\"\n :fields #{(merge base-field\n {:name \"idx\"\n :database-type \"UInt8\"\n :base-type :type/Integer\n :database-position 0})\n (merge base-field\n {:name \"lowest_value\"\n :database-type \"SimpleAggregateFunction(min, UInt8)\"\n :base-type :type/Integer\n :database-position 2})\n (merge base-field\n {:name \"count\"\n :database-type \"SimpleAggregateFunction(sum, Int64)\"\n :base-type :type/BigInteger\n :database-position 3})}}\n (ctd/do-with-test-db\n (fn [db]\n (driver/describe-table :clickhouse db {:name \"aggregate_functions_filter_test\"})))))))\n\n(deftest ^:parallel clickhouse-filtered-aggregate-functions-test-result-set\n (mt/test-driver\n :clickhouse\n (is (= [[42 144 255255]]\n (qp.test/formatted-rows\n [int int int]\n :format-nil-values\n (ctd/do-with-test-db\n (fn [db]\n (data/with-db db\n (data/run-mbql-query\n aggregate_functions_filter_test\n {})))))))))\n\n(def ^:private test-tables\n #{{:description nil,\n :name \"table1\",\n :schema \"metabase_db_scan_test\"}\n {:description nil,\n :name \"table2\",\n :schema \"metabase_db_scan_test\"}})\n\n(deftest ^:parallel clickhouse-describe-database-single\n (mt/test-driver\n :clickhouse\n (t2.with-temp/with-temp\n [:model/Database db {:engine :clickhouse\n :details (merge {:scan-all-databases nil}\n (tx/dbdef->connection-details\n :clickhouse :db\n {:database-name \"metabase_db_scan_test\"}))}]\n (let [describe-result (driver/describe-database :clickhouse db)]\n (is (= {:tables test-tables} describe-result))))))\n\n(deftest ^:parallel clickhouse-describe-database-all\n (mt/test-driver\n :clickhouse\n (t2.with-temp/with-temp\n [:model/Database db {:engine :clickhouse\n :details (merge {:scan-all-databases true}\n (tx/dbdef->connection-details\n :clickhouse :db\n {:database-name \"default\"}))}]\n (let [describe-result (driver/describe-database :clickhouse db)]\n ;; check the existence of at least some test tables here\n (doseq [table test-tables]\n (is (contains? (:tables describe-result) table)))\n ;; should not contain any ClickHouse system tables\n (is (not (some #(= (:schema %) \"system\")\n (:tables describe-result))))\n (is (not (some #(= (:schema %) \"information_schema\")\n (:tables describe-result))))\n (is (not (some #(= (:schema %) \"INFORMATION_SCHEMA\")\n (:tables describe-result))))))))\n\n(deftest ^:parallel clickhouse-describe-database-multiple\n (mt/test-driver\n :clickhouse\n (t2.with-temp/with-temp\n [:model/Database db {:engine :clickhouse\n :details (tx/dbdef->connection-details\n :clickhouse :db\n {:database-name \"metabase_db_scan_test information_schema\"})}]\n (let [{:keys [tables] :as _describe-result}\n (driver/describe-database :clickhouse db)\n tables-table {:name \"tables\"\n :description nil\n :schema \"information_schema\"}\n columns-table {:name \"columns\"\n :description nil\n :schema \"information_schema\"}]\n\n ;; tables from `metabase_db_scan_test`\n (doseq [table test-tables]\n (is (contains? tables table)))\n\n ;; tables from `information_schema`\n (is (contains? tables tables-table))\n (is (contains? tables columns-table))))))\n" }, { "path": "test/metabase/driver/clickhouse_substitution_test.clj", "content": "(ns metabase.driver.clickhouse-substitution-test\n #_{:clj-kondo/ignore [:unsorted-required-namespaces]}\n (:require [clojure.test :refer :all]\n [java-time.api :as t]\n [metabase.query-processor :as qp]\n [metabase.test :as mt]\n [metabase.test.data :as data]\n [metabase.test.data.interface :as tx]\n [metabase.test.data.clickhouse :as ctd]\n [metabase.util :as u]\n [schema.core :as s])\n (:import (java.time LocalDate LocalDateTime)))\n\n(set! *warn-on-reflection* true)\n\n(defn- get-mbql\n [value db]\n (let [uuid (str (java.util.UUID/randomUUID))]\n {:database (mt/id)\n :type \"native\"\n :native {:collection \"test-table\"\n :template-tags\n {:x {:id uuid\n :name \"d\"\n :display-name \"D\"\n :type \"dimension\"\n :dimension [\"field\" (mt/id :test-table :d) nil]\n :required true}}\n :query (format \"SELECT * FROM `%s`.`test_table` WHERE {{x}}\" db)}\n :parameters [{:type \"date/all-options\"\n :value value\n :target [\"dimension\" [\"template-tag\" \"x\"]]\n :id uuid}]}))\n\n(def ^:private clock (t/mock-clock (t/instant \"2019-11-30T23:00:00Z\") (t/zone-id \"UTC\")))\n(s/defn ^:private local-date-now :- LocalDate [] (LocalDate/now clock))\n(s/defn ^:private local-date-time-now :- LocalDateTime [] (LocalDateTime/now clock))\n\n(deftest ^:parallel clickhouse-variables-field-filters-datetime-and-datetime64\n (mt/test-driver\n :clickhouse\n (mt/with-clock clock\n (letfn\n [(->clickhouse-input\n [^LocalDateTime ldt]\n [(t/format \"yyyy-MM-dd HH:mm:ss\" ldt)])\n (get-test-table\n [rows native-type]\n [\"test_table\"\n [{:field-name \"d\"\n :base-type {:native native-type}}]\n (map ->clickhouse-input rows)])\n (->iso-str\n [^LocalDateTime ldt]\n (t/format \"yyyy-MM-dd'T'HH:mm:ss'Z'\" ldt))]\n (doseq [base-type [\"DateTime\" \"DateTime64\"]]\n (testing base-type\n (testing \"on specific\"\n (let [db (format \"mb_vars_on_x_%s\"\n (u/lower-case-en base-type))\n now (local-date-time-now)\n row1 (.minusHours now 14)\n row2 (.minusMinutes now 20)\n row3 (.plusMinutes now 5)\n row4 (.plusHours now 6)\n table (get-test-table [row1 row2 row3 row4] base-type)]\n (data/dataset\n (tx/dataset-definition db table)\n (testing \"date\"\n (is (= [[(->iso-str row1)] [(->iso-str row2)] [(->iso-str row3)]]\n (ctd/rows-without-index (qp/process-query (get-mbql \"2019-11-30\" db))))))\n (testing \"datetime\"\n (is (= [[(->iso-str row2)]]\n (ctd/rows-without-index (qp/process-query (get-mbql \"2019-11-30T22:40:00\" db)))))))))\n (testing \"past/next minutes\"\n (let [db (format \"mb_vars_m_%s\"\n (u/lower-case-en base-type))\n now (local-date-time-now)\n row1 (.minusHours now 14)\n row2 (.minusMinutes now 20)\n row3 (.plusMinutes now 5)\n row4 (.plusHours now 6)\n table (get-test-table [row1 row2 row3 row4] base-type)]\n (data/dataset\n (tx/dataset-definition db table)\n (testing \"past30minutes\"\n (is (= [[(->iso-str row2)]]\n (ctd/rows-without-index (qp/process-query (get-mbql \"past30minutes\" db))))))\n (testing \"next30minutes\"\n (is (= [[(->iso-str row3)]]\n (ctd/rows-without-index (qp/process-query (get-mbql \"next30minutes\" db)))))))))\n (testing \"past/next hours\"\n (let [db (format \"mb_vars__past_next_hours_%s\"\n (u/lower-case-en base-type))\n now (local-date-time-now)\n row1 (.minusHours now 14)\n row2 (.minusHours now 2)\n row3 (.plusHours now 25)\n row4 (.plusHours now 6)\n table (get-test-table [row1 row2 row3 row4] base-type)]\n (data/dataset\n (tx/dataset-definition db table)\n (testing \"past12hours\"\n (is (= [[(->iso-str row2)]]\n (ctd/rows-without-index (qp/process-query (get-mbql \"past12hours\" db))))))\n (testing \"next12hours\"\n (is (= [[(->iso-str row4)]]\n (ctd/rows-without-index (qp/process-query (get-mbql \"next12hours\" db)))))))))\n (testing \"past/next days\"\n (let [db (format \"mb_vars_d_%s\"\n (u/lower-case-en base-type))\n now (local-date-time-now)\n row1 (.minusDays now 14)\n row2 (.minusDays now 2)\n row3 (.plusDays now 25)\n row4 (.plusDays now 6)\n table (get-test-table [row1 row2 row3 row4] base-type)]\n (data/dataset\n (tx/dataset-definition db table)\n (testing \"past12days\"\n (is (= [[(->iso-str row2)]]\n (ctd/rows-without-index (qp/process-query (get-mbql \"past12days\" db))))))\n (testing \"next12days\"\n (is (= [[(->iso-str row4)]]\n (ctd/rows-without-index (qp/process-query (get-mbql \"next12days\" db)))))))))\n (testing \"past/next months/quarters\"\n (let [db (format \"mb_vars_m_q_%s\"\n (u/lower-case-en base-type))\n now (local-date-time-now)\n row1 (.minusMonths now 14)\n row2 (.minusMonths now 4)\n row3 (.plusMonths now 25)\n row4 (.plusMonths now 6)\n table (get-test-table [row1 row2 row3 row4] base-type)]\n (data/dataset\n (tx/dataset-definition db table)\n (testing \"past12months\"\n (is (= [[(->iso-str row2)]]\n (ctd/rows-without-index (qp/process-query (get-mbql \"past12months\" db))))))\n (testing \"next12months\"\n (is (= [[(->iso-str row4)]]\n (ctd/rows-without-index (qp/process-query (get-mbql \"next12months\" db))))))\n (testing \"past3quarters\"\n (is (= [[(->iso-str row2)]]\n (ctd/rows-without-index (qp/process-query (get-mbql \"past3quarters\" db))))))\n (testing \"next3quarters\"\n (is (= [[(->iso-str row4)]]\n (ctd/rows-without-index (qp/process-query (get-mbql \"next3quarters\" db)))))))))\n (testing \"past/next years\"\n (let [db (format \"mb_vars_y_%s\"\n (u/lower-case-en base-type))\n now (local-date-time-now)\n row1 (.minusYears now 14)\n row2 (.minusYears now 4)\n row3 (.plusYears now 25)\n row4 (.plusYears now 6)\n table (get-test-table [row1 row2 row3 row4] base-type)]\n (data/dataset\n (tx/dataset-definition db table)\n (testing \"past12years\"\n (is (= [[(->iso-str row2)]]\n (ctd/rows-without-index (qp/process-query (get-mbql \"past12years\" db))))))\n (testing \"next12years\"\n (is (= [[(->iso-str row4)]]\n (ctd/rows-without-index (qp/process-query (get-mbql \"next12years\" db)))))))))))))))\n\n(deftest ^:parallel clickhouse-variables-field-filters-date-and-date32\n (mt/test-driver\n :clickhouse\n (mt/with-clock clock\n (letfn\n [(->clickhouse-input\n [^LocalDate ld]\n [(t/format \"yyyy-MM-dd\" ld)])\n (get-test-table\n [rows native-type]\n [\"test_table\"\n [{:field-name \"d\"\n :base-type {:native native-type}}]\n (map ->clickhouse-input rows)])\n (->iso-str\n [^LocalDate ld]\n (str (t/format \"yyyy-MM-dd\" ld) \"T00:00:00Z\"))]\n (doseq [base-type [\"Date\" \"Date32\"]]\n (testing base-type\n (testing \"on specific date\"\n (let [db (format \"mb_vars_on_x_%s\"\n (u/lower-case-en base-type))\n now (local-date-time-now)\n row1 (.minusDays now 14)\n row2 now\n row3 (.plusDays now 25)\n row4 (.plusDays now 6)\n table (get-test-table [row1 row2 row3 row4] base-type)]\n (data/dataset\n (tx/dataset-definition db table)\n (is (= [[(->iso-str row2)]]\n (ctd/rows-without-index (qp/process-query (get-mbql \"2019-11-30\" db))))))))\n (testing \"past/next days\"\n (let [db (format \"mb_vars_d_%s\"\n (u/lower-case-en base-type))\n now (local-date-now)\n row1 (.minusDays now 14)\n row2 (.minusDays now 2)\n row3 (.plusDays now 25)\n row4 (.plusDays now 6)\n table (get-test-table [row1 row2 row3 row4] base-type)]\n (data/dataset\n (tx/dataset-definition db table)\n (testing \"past12days\"\n (is (= [[(->iso-str row2)]]\n (ctd/rows-without-index (qp/process-query (get-mbql \"past12days\" db))))))\n (testing \"next12days\"\n (is (= [[(->iso-str row4)]]\n (ctd/rows-without-index (qp/process-query (get-mbql \"next12days\" db)))))))))\n (testing \"past/next months/quarters\"\n (let [db (format \"mb_vars_m_q_%s\"\n (u/lower-case-en base-type))\n now (local-date-now)\n row1 (.minusMonths now 14)\n row2 (.minusMonths now 4)\n row3 (.plusMonths now 25)\n row4 (.plusMonths now 6)\n table (get-test-table [row1 row2 row3 row4] base-type)]\n (data/dataset\n (tx/dataset-definition db table)\n (testing \"past12months\"\n (is (= [[(->iso-str row2)]]\n (ctd/rows-without-index (qp/process-query (get-mbql \"past12months\" db))))))\n (testing \"next12months\"\n (is (= [[(->iso-str row4)]]\n (ctd/rows-without-index (qp/process-query (get-mbql \"next12months\" db))))))\n (testing \"past3quarters\"\n (is (= [[(->iso-str row2)]]\n (ctd/rows-without-index (qp/process-query (get-mbql \"past3quarters\" db))))))\n (testing \"next3quarters\"\n (is (= [[(->iso-str row4)]]\n (ctd/rows-without-index (qp/process-query (get-mbql \"next3quarters\" db)))))))))\n (testing \"past/next years\"\n (let [db (format \"mb_vars_y_%s\"\n (u/lower-case-en base-type))\n now (local-date-now)\n row1 (.minusYears now 14)\n row2 (.minusYears now 4)\n row3 (.plusYears now 25)\n row4 (.plusYears now 6)\n table (get-test-table [row1 row2 row3 row4] base-type)]\n (data/dataset\n (tx/dataset-definition db table)\n (testing \"past12years\"\n (is (= [[(->iso-str row2)]]\n (ctd/rows-without-index (qp/process-query (get-mbql \"past12years\" db))))))\n (testing \"next12years\"\n (is (= [[(->iso-str row4)]]\n (ctd/rows-without-index (qp/process-query (get-mbql \"next12years\" db)))))))))))))))\n\n(deftest ^:parallel clickhouse-variables-field-filters-null-dates\n (mt/test-driver\n :clickhouse\n (mt/with-clock clock\n (letfn\n [(->input-ld\n [^LocalDate ld]\n [(t/format \"yyyy-MM-dd\" ld)])\n (->input-ldt\n [^LocalDateTime ldt]\n [(t/format \"yyyy-MM-dd HH:mm:ss\" ldt)])\n (->iso-str-ld\n [^LocalDate ld]\n (str (t/format \"yyyy-MM-dd\" ld) \"T00:00:00Z\"))\n (->iso-str-ldt\n [^LocalDateTime ldt]\n (t/format \"yyyy-MM-dd'T'HH:mm:ss'Z'\" ldt))]\n (let [db \"mb_vars_null_dates\"\n now-ld (local-date-now)\n now-ldt (local-date-time-now)\n table [\"test_table\"\n [{:field-name \"d\"\n :base-type {:native \"Date\"}}\n {:field-name \"d32\"\n :base-type {:native \"Date32\"}}\n {:field-name \"dt\"\n :base-type {:native \"DateTime\"}}\n {:field-name \"dt64\"\n :base-type {:native \"DateTime64\"}}]\n [;; row 1\n [(->input-ld now-ld) nil (->input-ldt now-ldt) nil]\n ;; row 2\n [nil (->input-ld now-ld) nil (->input-ldt now-ldt)]]]\n first-row [[(->iso-str-ld now-ld) nil (->iso-str-ldt now-ldt) nil]]\n second-row [[nil (->iso-str-ld now-ld) nil (->iso-str-ldt now-ldt)]]]\n (data/dataset\n (tx/dataset-definition db table)\n (letfn\n [(get-mbql*\n [field value]\n (let [uuid (str (java.util.UUID/randomUUID))]\n {:database (mt/id)\n :type \"native\"\n :native {:collection \"test-table\"\n :template-tags\n {:x {:id uuid\n :name (str field)\n :display-name (str field)\n :type \"dimension\"\n :dimension [\"field\" (mt/id :test-table field) nil]\n :required true}}\n :query (format \"SELECT * FROM `%s`.`test_table` WHERE {{x}}\" db)}\n :parameters [{:type \"date/all-options\"\n :value value\n :target [\"dimension\" [\"template-tag\" \"x\"]]\n :id uuid}]}))]\n (testing \"first row (Date field match)\"\n (is (= first-row (ctd/rows-without-index (qp/process-query (get-mbql* :d \"2019-11-30\"))))))\n (testing \"first row (DateTime field match)\"\n (is (= first-row (ctd/rows-without-index (qp/process-query (get-mbql* :dt \"2019-11-30T23:00:00\"))))))\n (testing \"second row (Date32 field match)\"\n (is (= second-row (ctd/rows-without-index (qp/process-query (get-mbql* :d32 \"2019-11-30\"))))))\n (testing \"second row (DateTime64 field match)\"\n (is (= second-row (ctd/rows-without-index (qp/process-query (get-mbql* :dt64 \"2019-11-30T23:00:00\")))))))))))))\n" }, { "path": "test/metabase/driver/clickhouse_temporal_bucketing_test.clj", "content": "(ns metabase.driver.clickhouse-temporal-bucketing-test\n #_{:clj-kondo/ignore [:unsorted-required-namespaces]}\n (:require\n [clojure.test :refer :all]\n [metabase.query-processor.test-util :as qp.test]\n [metabase.test :as mt]\n [metabase.test.data :as data]\n [metabase.test.data.clickhouse :as ctd]))\n\n(use-fixtures :once ctd/create-test-db!)\n\n;; See temporal_bucketing table definition\n;; Fields values are (both in server and column timezones):\n;; start_of_year == '2022-01-01 00:00:00'\n;; mid_of_year == '2022-06-20 06:32:54'\n;; end_of_year == '2022-12-31 23:59:59'\n(deftest clickhouse-temporal-bucketing-server-tz\n (mt/test-driver\n :clickhouse\n (defn- start-of-year [unit]\n (qp.test/rows\n (ctd/do-with-test-db\n (fn [db]\n (data/with-db db\n (data/run-mbql-query\n temporal_bucketing_server_tz\n {:breakout [[:field %start_of_year {:temporal-unit unit}]]}))))))\n (defn- mid-year [unit]\n (qp.test/rows\n (ctd/do-with-test-db\n (fn [db]\n (data/with-db db\n (data/run-mbql-query\n temporal_bucketing_server_tz\n {:breakout [[:field %mid_of_year {:temporal-unit unit}]]}))))))\n (defn- end-of-year [unit]\n (qp.test/rows\n (ctd/do-with-test-db\n (fn [db]\n (data/with-db db\n (data/run-mbql-query\n temporal_bucketing_server_tz\n {:breakout [[:field %end_of_year {:temporal-unit unit}]]}))))))\n (testing \"truncate to\"\n (testing \"minute\"\n (is (= [[\"2022-06-20T06:32:00Z\"]]\n (mid-year :minute))))\n (testing \"hour\"\n (is (= [[\"2022-06-20T06:00:00Z\"]]\n (mid-year :hour))))\n (testing \"day\"\n (is (= [[\"2022-06-20T00:00:00Z\"]]\n (mid-year :day))))\n (testing \"month\"\n (is (= [[\"2022-06-01T00:00:00Z\"]]\n (mid-year :month))))\n (testing \"quarter\"\n (is (= [[\"2022-04-01T00:00:00Z\"]]\n (mid-year :quarter))))\n (testing \"year\"\n (is (= [[\"2022-01-01T00:00:00Z\"]]\n (mid-year :year)))))\n (testing \"extract\"\n (testing \"minute of hour\"\n (is (= [[0]]\n (start-of-year :minute-of-hour)))\n (is (= [[32]]\n (mid-year :minute-of-hour)))\n (is (= [[59]]\n (end-of-year :minute-of-hour))))\n (testing \"hour of day\"\n (is (= [[0]]\n (start-of-year :hour-of-day)))\n (is (= [[6]]\n (mid-year :hour-of-day)))\n (is (= [[23]]\n (end-of-year :hour-of-day))))\n (testing \"day of month\"\n (is (= [[1]]\n (start-of-year :day-of-month)))\n (is (= [[20]]\n (mid-year :day-of-month)))\n (is (= [[31]]\n (end-of-year :day-of-month))))\n (testing \"day of year\"\n (is (= [[1]]\n (start-of-year :day-of-year)))\n (is (= [[171]]\n (mid-year :day-of-year)))\n (is (= [[365]]\n (end-of-year :day-of-year))))\n (testing \"month of year\"\n (is (= [[1]]\n (start-of-year :month-of-year)))\n (is (= [[6]]\n (mid-year :month-of-year)))\n (is (= [[12]]\n (end-of-year :month-of-year))))\n (testing \"quarter of year\"\n (is (= [[1]]\n (start-of-year :quarter-of-year)))\n (is (= [[2]]\n (mid-year :quarter-of-year)))\n (is (= [[4]]\n (end-of-year :quarter-of-year)))))))\n\n(deftest clickhouse-temporal-bucketing-column-tz\n (mt/test-driver\n :clickhouse\n (defn- start-of-year [unit]\n (qp.test/rows\n (ctd/do-with-test-db\n (fn [db]\n (data/with-db db\n (data/run-mbql-query\n temporal_bucketing_column_tz\n {:breakout [[:field %start_of_year {:temporal-unit unit}]]}))))))\n (defn- mid-year [unit]\n (qp.test/rows\n (ctd/do-with-test-db\n (fn [db]\n (data/with-db db\n (data/run-mbql-query\n temporal_bucketing_column_tz\n {:breakout [[:field %mid_of_year {:temporal-unit unit}]]}))))))\n (defn- end-of-year [unit]\n (qp.test/rows\n (ctd/do-with-test-db\n (fn [db]\n (data/with-db db\n (data/run-mbql-query\n temporal_bucketing_column_tz\n {:breakout [[:field %end_of_year {:temporal-unit unit}]]}))))))\n (testing \"truncate to\"\n (testing \"minute\"\n (is (= [[\"2022-06-20T13:32:00Z\"]]\n (mid-year :minute))))\n (testing \"hour\"\n (is (= [[\"2022-06-20T13:00:00Z\"]]\n (mid-year :hour))))\n (testing \"day\"\n (is (= [[\"2022-06-20T07:00:00Z\"]]\n (mid-year :day))))\n (testing \"month\"\n (is (= [[\"2022-06-01T00:00:00Z\"]]\n (mid-year :month))))\n (testing \"quarter\"\n (is (= [[\"2022-04-01T00:00:00Z\"]]\n (mid-year :quarter))))\n (testing \"year\"\n (is (= [[\"2022-01-01T00:00:00Z\"]]\n (mid-year :year)))))\n (testing \"extract\"\n (testing \"minute of hour\"\n (is (= [[0]]\n (start-of-year :minute-of-hour)))\n (is (= [[32]]\n (mid-year :minute-of-hour)))\n (is (= [[59]]\n (end-of-year :minute-of-hour))))\n (testing \"hour of day\"\n (is (= [[0]]\n (start-of-year :hour-of-day)))\n (is (= [[6]]\n (mid-year :hour-of-day)))\n (is (= [[23]]\n (end-of-year :hour-of-day))))\n (testing \"day of month\"\n (is (= [[1]]\n (start-of-year :day-of-month)))\n (is (= [[20]]\n (mid-year :day-of-month)))\n (is (= [[31]]\n (end-of-year :day-of-month))))\n (testing \"day of year\"\n (is (= [[1]]\n (start-of-year :day-of-year)))\n (is (= [[171]]\n (mid-year :day-of-year)))\n (is (= [[365]]\n (end-of-year :day-of-year))))\n (testing \"month of year\"\n (is (= [[1]]\n (start-of-year :month-of-year)))\n (is (= [[6]]\n (mid-year :month-of-year)))\n (is (= [[12]]\n (end-of-year :month-of-year))))\n (testing \"quarter of year\"\n (is (= [[1]]\n (start-of-year :quarter-of-year)))\n (is (= [[2]]\n (mid-year :quarter-of-year)))\n (is (= [[4]]\n (end-of-year :quarter-of-year)))))))\n" }, { "path": "test/metabase/driver/clickhouse_test.clj", "content": "(ns metabase.driver.clickhouse-test\n \"Tests for specific behavior of the ClickHouse driver.\"\n #_{:clj-kondo/ignore [:unsorted-required-namespaces]}\n (:require [clojure.test :refer :all]\n [metabase.driver :as driver]\n [metabase.driver.clickhouse :as clickhouse]\n [metabase.driver.clickhouse-qp :as clickhouse-qp]\n [metabase.driver.sql-jdbc :as sql-jdbc]\n [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]\n [metabase.query-processor.compile :as qp.compile]\n [metabase.test :as mt]\n [metabase.test.data :as data]\n [metabase.test.data.interface :as tx]\n [metabase.test.data.clickhouse :as ctd]\n [taoensso.nippy :as nippy]\n [toucan2.tools.with-temp :as t2.with-temp]))\n\n(set! *warn-on-reflection* true)\n\n(use-fixtures :once ctd/create-test-db!)\n\n;; the mt/with-dynamic-redefs macro was renamed to mt/with-dynamic-fn-redefs for 0.53+\n;; as 0.52 is still tested by CI we will check which macro is defined and use that\n(defmacro with-dynamic-redefs [bindings & body]\n (if (resolve `mt/with-dynamic-redefs)\n `(mt/with-dynamic-redefs ~bindings ~@body)\n `(mt/with-dynamic-fn-redefs ~bindings ~@body)))\n\n(deftest ^:parallel clickhouse-version\n (mt/test-driver\n :clickhouse\n (t2.with-temp/with-temp\n [:model/Database db\n {:engine :clickhouse\n :details (tx/dbdef->connection-details :clickhouse :db {:database-name \"default\"})}]\n (let [version (driver/dbms-version :clickhouse db)]\n (is (number? (get-in version [:semantic-version :major])))\n (is (number? (get-in version [:semantic-version :minor])))\n (is (string? (get version :version)))))))\n\n(deftest ^:parallel clickhouse-server-timezone\n (mt/test-driver\n :clickhouse\n (is (= \"UTC\"\n (let [details (tx/dbdef->connection-details :clickhouse :db {:database-name \"default\"})\n spec (sql-jdbc.conn/connection-details->spec :clickhouse details)]\n (driver/db-default-timezone :clickhouse spec))))))\n\n(deftest ^:parallel clickhouse-connection-string\n (mt/with-dynamic-fn-redefs [ ;; This function's implementation requires the connection details to actually connect to the\n ;; database, which is orthogonal to the purpose of this test.\n clickhouse/cloud? (constantly false)]\n (testing \"connection with no additional options\"\n (is (= ctd/default-connection-params\n (sql-jdbc.conn/connection-details->spec\n :clickhouse\n {}))))\n (testing \"custom connection with additional options\"\n (is (= (merge\n ctd/default-connection-params\n {:subname \"//myclickhouse:9999/foo?sessionTimeout=42\"\n :user \"bob\"\n :password \"qaz\"\n :ssl true\n :custom_http_params \"max_threads=42,allow_experimental_analyzer=0\"})\n (sql-jdbc.conn/connection-details->spec\n :clickhouse\n {:host \"myclickhouse\"\n :port 9999\n :user \"bob\"\n :password \"qaz\"\n :dbname \"foo\"\n :additional-options \"sessionTimeout=42\"\n :ssl true\n :clickhouse-settings \"max_threads=42,allow_experimental_analyzer=0\"}))))\n (testing \"nil dbname handling\"\n (is (= ctd/default-connection-params\n (sql-jdbc.conn/connection-details->spec\n :clickhouse {:dbname nil}))))\n (testing \"schema removal\"\n (doall\n (for [host [\"localhost\" \"http://localhost\" \"https://localhost\"]]\n (testing (str \"for host \" host)\n (is (= ctd/default-connection-params\n (sql-jdbc.conn/connection-details->spec\n :clickhouse {:host host}))))))\n (doall\n (for [host [\"myhost\" \"http://myhost\" \"https://myhost\"]]\n (testing (str \"for host \" host)\n (is (= (merge ctd/default-connection-params\n {:subname \"//myhost:8123/default\"})\n (sql-jdbc.conn/connection-details->spec\n :clickhouse {:host host}))))))\n (doall\n (for [host [\"sub.example.com\" \"http://sub.example.com\" \"https://sub.example.com\"]]\n (testing (str \"for host \" host \" with some additional params\")\n (is (= (merge ctd/default-connection-params\n {:subname \"//sub.example.com:8443/mydb\" :ssl true})\n (sql-jdbc.conn/connection-details->spec\n :clickhouse {:host host :dbname \"mydb\" :port 8443 :ssl true})))))))))\n\n(deftest ^:parallel clickhouse-connection-string-select-sequential-consistency\n (mt/with-dynamic-fn-redefs [ ;; This function's implementation requires the connection details to actually\n ;; connect to the database, which is orthogonal to the purpose of this test.\n clickhouse/cloud? (constantly true)]\n (testing \"connection with no additional options\"\n (is (= (assoc ctd/default-connection-params :select_sequential_consistency true)\n (sql-jdbc.conn/connection-details->spec\n :clickhouse\n {}))))))\n\n(deftest clickhouse-connection-fails-test\n (mt/test-driver\n :clickhouse\n (mt/with-temp [:model/Database db {:details (assoc (mt/db) :password \"wrongpassword\") :engine :clickhouse}]\n (testing \"sense check that checking the cloud mode fails with a SQLException.\"\n ;; nil arg isn't tested here, as it will pick up the defaults, which is the same as the Docker instance credentials.\n (is (thrown? java.sql.SQLException (#'clickhouse/cloud? (:details db)))))\n (testing \"`driver/database-supports? :uploads` does not throw even if the connection fails.\"\n (is (false? (driver/database-supports? :clickhouse :uploads db)))\n (is (false? (driver/database-supports? :clickhouse :uploads nil))))\n (testing \"`driver/database-supports? :connection-impersonation` does not throw even if the connection fails.\"\n (is (false? (driver/database-supports? :clickhouse :connection-impersonation db)))\n (is (false? (driver/database-supports? :clickhouse :connection-impersonation nil))))\n (testing (str \"`sql-jdbc.conn/connection-details->spec` does not throw even if the connection fails, \"\n \"and doesn't include the `select_sequential_consistency` parameter.\")\n (is (nil? (:select_sequential_consistency (sql-jdbc.conn/connection-details->spec :clickhouse db))))\n (is (nil? (:select_sequential_consistency (sql-jdbc.conn/connection-details->spec :clickhouse nil))))))))\n\n(deftest ^:parallel clickhouse-tls\n (mt/test-driver\n :clickhouse\n (let [working-dir (System/getProperty \"user.dir\")\n cert-path (str working-dir \"/modules/drivers/clickhouse/.docker/clickhouse/single_node_tls/certificates/ca.crt\")\n additional-options (str \"sslrootcert=\" cert-path)]\n (testing \"simple connection with a single database\"\n (is (= \"UTC\"\n (driver/db-default-timezone\n :clickhouse\n (sql-jdbc.conn/connection-details->spec\n :clickhouse\n {:ssl true\n :host \"server.clickhouseconnect.test\"\n :port 8443\n :additional-options additional-options})))))\n (testing \"connection with multiple databases\"\n (is (= \"UTC\"\n (driver/db-default-timezone\n :clickhouse\n (sql-jdbc.conn/connection-details->spec\n :clickhouse\n {:ssl true\n :host \"server.clickhouseconnect.test\"\n :port 8443\n :dbname \"default system\"\n :additional-options additional-options}))))))))\n\n(deftest ^:parallel clickhouse-nippy\n (mt/test-driver\n :clickhouse\n (testing \"UnsignedByte\"\n (let [value (com.clickhouse.data.value.UnsignedByte/valueOf \"214\")]\n (is (= value (nippy/thaw (nippy/freeze value))))))\n (testing \"UnsignedShort\"\n (let [value (com.clickhouse.data.value.UnsignedShort/valueOf \"62055\")]\n (is (= value (nippy/thaw (nippy/freeze value))))))\n (testing \"UnsignedInteger\"\n (let [value (com.clickhouse.data.value.UnsignedInteger/valueOf \"4748364\")]\n (is (= value (nippy/thaw (nippy/freeze value))))))\n (testing \"UnsignedLong\"\n (let [value (com.clickhouse.data.value.UnsignedLong/valueOf \"84467440737095\")]\n (is (= value (nippy/thaw (nippy/freeze value))))))))\n\n(deftest ^:parallel clickhouse-query-formatting\n (mt/test-driver\n :clickhouse\n (let [query (data/mbql-query venues {:fields [$id] :order-by [[:asc $id]] :limit 5})\n {compiled :query} (qp.compile/compile-with-inline-parameters query)\n pretty (driver/prettify-native-form :clickhouse compiled)]\n (testing \"compiled\"\n (is (= \"SELECT `test_data`.`venues`.`id` AS `id` FROM `test_data`.`venues` ORDER BY `test_data`.`venues`.`id` ASC LIMIT 5\" compiled)))\n (testing \"pretty\"\n (is (= \"SELECT\\n `test_data`.`venues`.`id` AS `id`\\nFROM\\n `test_data`.`venues`\\nORDER BY\\n `test_data`.`venues`.`id` ASC\\nLIMIT\\n 5\" pretty))))))\n\n(deftest ^:parallel clickhouse-can-connect\n (mt/test-driver\n :clickhouse\n (doall\n (for [[username password] [[\"default\" \"\"] [\"user_with_password\" \"foo@bar!\"]]\n database [\"default\" \"Special@Characters~\"]]\n (testing (format \"User `%s` can connect to `%s`\" username database)\n (let [details (merge {:user username :password password}\n (tx/dbdef->connection-details :clickhouse :db {:database-name database}))]\n (is (true? (driver/can-connect? :clickhouse details)))))))))\n\n(deftest clickhouse-qp-extract-datetime-timezone\n (mt/test-driver\n :clickhouse\n (is (= \"utc\" (#'clickhouse-qp/extract-datetime-timezone \"datetime('utc')\")))\n (is (= \"utc\" (#'clickhouse-qp/extract-datetime-timezone \"datetime64(3, 'utc')\")))\n (is (= \"europe/amsterdam\" (#'clickhouse-qp/extract-datetime-timezone \"datetime('europe/amsterdam')\")))\n (is (= \"europe/amsterdam\" (#'clickhouse-qp/extract-datetime-timezone \"datetime64(9, 'europe/amsterdam')\")))\n (is (= nil (#'clickhouse-qp/extract-datetime-timezone \"datetime\")))\n (is (= nil (#'clickhouse-qp/extract-datetime-timezone \"datetime64\")))\n (is (= nil (#'clickhouse-qp/extract-datetime-timezone \"datetime64(3)\")))))\n\n(deftest ^:synchronized clickhouse-insert\n (mt/test-driver\n :clickhouse\n (t2.with-temp/with-temp\n [:model/Database db\n {:engine :clickhouse\n :details (tx/dbdef->connection-details :clickhouse :db {:database-name \"default\"})}]\n (let [table (keyword (format \"insert_table_%s\" (System/currentTimeMillis)))]\n (driver/create-table! :clickhouse (:id db) table {:id \"Int64\", :name \"String\"})\n (try\n (driver/insert-into! :clickhouse (:id db) table [:id :name] [[42 \"Bob\"] [43 \"Alice\"]])\n (is (= #{{:id 42, :name \"Bob\"}\n {:id 43, :name \"Alice\"}}\n (set (sql-jdbc/query :clickhouse db {:select [:*] :from [table]}))))\n (finally\n (driver/drop-table! :clickhouse (:id db) table)))))))\n" }, { "path": "test/metabase/test/data/clickhouse.clj", "content": "(ns metabase.test.data.clickhouse\n \"Code for creating / destroying a ClickHouse database from a `DatabaseDefinition`.\"\n (:require\n [clojure.java.io :as io]\n [clojure.java.jdbc :as jdbc]\n [clojure.string :as str]\n [clojure.test :refer :all]\n [java-time.api :as t]\n [metabase.db.query :as mdb.query]\n [metabase.driver :as driver]\n [metabase.driver.ddl.interface :as ddl.i]\n [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]\n [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]\n [metabase.driver.sql.util :as sql.u]\n [metabase.lib.schema.common :as lib.schema.common]\n [metabase.query-processor-test.alternative-date-test :as qp.alternative-date-test]\n [metabase.query-processor.test-util :as qp.test]\n [metabase.sync.core :as sync]\n [metabase.test.data.interface :as tx]\n [metabase.test.data.sql :as sql.tx]\n [metabase.test.data.sql-jdbc :as sql-jdbc.tx]\n [metabase.test.data.sql-jdbc.execute :as execute]\n [metabase.test.data.sql-jdbc.load-data :as load-data]\n [metabase.test.data.sql.ddl :as ddl]\n [metabase.util.log :as log]\n [metabase.util.malli :as mu]\n [toucan2.tools.with-temp :as t2.with-temp]))\n\n(sql-jdbc.tx/add-test-extensions! :clickhouse)\n\n(defmethod driver/database-supports? [:clickhouse :metabase.driver.sql-jdbc.sync.describe-table-test/describe-view-fields]\n [_driver _feature _db] true)\n(defmethod driver/database-supports? [:clickhouse :metabase.driver.sql-jdbc.sync.describe-table-test/describe-materialized-view-fields]\n [_driver _feature _db] false)\n\n(defmethod driver/database-supports? [:clickhouse :metabase.query-processor-test.parameters-test/get-parameter-count]\n [_driver _feature _db] false)\n\n(defmethod qp.alternative-date-test/iso-8601-text-fields-expected-rows :clickhouse\n [_driver]\n [[1 \"foo\" (t/offset-date-time \"2004-10-19T10:23:54Z\") #t \"2004-10-19\" (t/offset-date-time \"1970-01-01T10:23:54Z\")]\n [2 \"bar\" (t/offset-date-time \"2008-10-19T10:23:54Z\") #t \"2008-10-19\" (t/offset-date-time \"1970-01-01T10:23:54Z\")]\n [3 \"baz\" (t/offset-date-time \"2012-10-19T10:23:54Z\") #t \"2012-10-19\" (t/offset-date-time \"1970-01-01T10:23:54Z\")]])\n\n(def default-connection-params\n {:classname \"com.clickhouse.jdbc.ClickHouseDriver\"\n :subprotocol \"clickhouse\"\n :subname \"//localhost:8123/default\"\n :user \"default\"\n :password \"\"\n :ssl false\n :use_server_time_zone_for_dates true\n :product_name \"metabase/1.53.4\"\n :jdbc_ignore_unsupported_values \"true\"\n :jdbc_schema_term \"schema\",\n :max_open_connections 100\n :remember_last_set_roles true\n :http_connection_provider \"HTTP_URL_CONNECTION\"\n :custom_http_params \"\"})\n\n(defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/Boolean] [_ _] \"Boolean\")\n(defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/BigInteger] [_ _] \"Int64\")\n(defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/Char] [_ _] \"String\")\n(defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/Date] [_ _] \"Date\")\n(defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/DateTime] [_ _] \"DateTime64(3, 'GMT0')\")\n(defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/DateTimeWithLocalTZ] [_ _] \"DateTime64(3, 'UTC')\")\n(defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/Float] [_ _] \"Float64\")\n(defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/Integer] [_ _] \"Int32\")\n(defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/IPAddress] [_ _] \"IPv4\")\n(defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/Text] [_ _] \"String\")\n(defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/UUID] [_ _] \"UUID\")\n(defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/Time] [_ _] \"Time\")\n\n(defmethod tx/sorts-nil-first? :clickhouse [_ _] false)\n\n(defmethod tx/dbdef->connection-details :clickhouse [_ context {:keys [database-name]}]\n (merge\n {:host (tx/db-test-env-var-or-throw :clickhouse :host \"localhost\")\n :port (tx/db-test-env-var-or-throw :clickhouse :port 8123)\n :timezone :America/Los_Angeles}\n (when-let [user (tx/db-test-env-var :clickhouse :user)]\n {:user user})\n (when-let [password (tx/db-test-env-var :clickhouse :password)]\n {:password password})\n (when (= context :db)\n {:db database-name})))\n\n(defmethod sql.tx/qualified-name-components :clickhouse\n ([_ db-name] [db-name])\n ([_ db-name table-name] [db-name table-name])\n ([_ db-name table-name field-name] [db-name table-name field-name]))\n\n(defmethod tx/create-db! :clickhouse\n [driver {:keys [database-name], :as db-def} & options]\n (let [database-name (ddl.i/format-name driver database-name)]\n (log/infof \"Creating ClickHouse database %s\" (pr-str database-name))\n ;; call the default impl for SQL JDBC drivers\n (apply (get-method tx/create-db! :sql-jdbc/test-extensions) driver db-def options)))\n\n(defmethod ddl/insert-rows-dml-statements :clickhouse\n [driver table-identifier rows]\n (binding [driver/*compile-with-inline-parameters* true]\n ((get-method ddl/insert-rows-dml-statements :sql-jdbc/test-extensions) driver table-identifier rows)))\n\n(mu/defmethod load-data/do-insert! :clickhouse\n [driver :- :keyword\n ^java.sql.Connection conn :- (lib.schema.common/instance-of-class java.sql.Connection)\n table-identifier\n rows]\n (let [statements (ddl/insert-rows-dml-statements driver table-identifier rows)]\n (doseq [sql-args statements\n :let [sql-args (if (string? sql-args)\n [sql-args]\n sql-args)]]\n (assert (string? (first sql-args))\n (format \"Bad sql-args: %s\" (pr-str sql-args)))\n (log/tracef \"[insert] %s\" (pr-str sql-args))\n (try\n (jdbc/execute! {:connection conn :transaction? false}\n sql-args\n {:set-parameters (fn [stmt params]\n (sql-jdbc.execute/set-parameters! driver stmt params))})\n (catch Throwable e\n (throw (ex-info (format \"INSERT FAILED: %s\" (ex-message e))\n {:driver driver\n :sql-args (into [(str/split-lines (mdb.query/format-sql (first sql-args)))]\n (rest sql-args))}\n e)))))))\n\n(defn- quote-name\n [name]\n (sql.u/quote-name :clickhouse :field (ddl.i/format-name :clickhouse name)))\n\n(def ^:private non-nullable-types [\"Array\" \"Map\" \"Tuple\"])\n(defn- disallowed-as-nullable?\n [ch-type]\n (boolean (some #(str/starts-with? ch-type %) non-nullable-types)))\n\n(defn- field->clickhouse-column\n [field]\n (let [{:keys [field-name base-type pk?]} field\n ch-type (if (map? base-type)\n (:native base-type)\n (sql.tx/field-base-type->sql-type :clickhouse base-type))\n col-name (quote-name field-name)\n ch-col (cond\n (or pk? (disallowed-as-nullable? ch-type))\n (format \"%s %s\" col-name ch-type)\n (= ch-type \"Time\")\n (format \"%s Nullable(DateTime64) COMMENT 'time'\" col-name)\n ; _\n :else (format \"%s Nullable(%s)\" col-name ch-type))]\n ch-col))\n\n(defn- ->comma-separated-str\n [coll]\n (->> coll\n (interpose \", \")\n (apply str)))\n\n(defmethod sql.tx/create-table-sql :clickhouse\n [_ {:keys [database-name]} {:keys [table-name field-definitions]}]\n (let [table-name (sql.tx/qualify-and-quote :clickhouse database-name table-name)\n pk-fields (filter (fn [{:keys [pk?]}] pk?) field-definitions)\n pk-field-names (map #(quote-name (:field-name %)) pk-fields)\n fields (->> field-definitions\n (map field->clickhouse-column)\n (->comma-separated-str))\n order-by (->comma-separated-str pk-field-names)]\n (format \"CREATE TABLE %s (%s)\n ENGINE = MergeTree\n ORDER BY (%s)\n SETTINGS allow_nullable_key=1\"\n table-name fields order-by)))\n\n(defmethod execute/execute-sql! :clickhouse [& args]\n (apply execute/sequentially-execute-sql! args))\n\n(defmethod load-data/row-xform :clickhouse [_driver _dbdef tabledef]\n (load-data/maybe-add-ids-xform tabledef))\n\n(defmethod sql.tx/pk-sql-type :clickhouse [_] \"Int32\")\n\n(defmethod sql.tx/add-fk-sql :clickhouse [& _] nil)\n\n(defmethod sql.tx/session-schema :clickhouse [_] \"default\")\n\n(defn rows-without-index\n \"Remove the Metabase index which is the first column in the result set\"\n [query-result]\n (map #(drop 1 %) (qp.test/rows query-result)))\n\n(def ^:private test-db-initialized? (atom false))\n(defn create-test-db!\n \"Create a ClickHouse database called `metabase_test` and initialize some test data\"\n [f]\n (when (not @test-db-initialized?)\n (let [details (tx/dbdef->connection-details :clickhouse :db {:database-name \"metabase_test\"})]\n ;; (println \"### Executing create-test-db! with details:\" details)\n (jdbc/with-db-connection\n [spec (sql-jdbc.conn/connection-details->spec :clickhouse (merge {:engine :clickhouse} details))]\n (let [raw-statements (slurp (io/resource \"metabase/test/data/clickhouse_datasets.sql\"))\n statements (as-> raw-statements s\n (str/split s #\";\")\n (map str/trim s)\n (filter seq s))]\n ;; (println \"## Executing statements \" statements)\n (jdbc/db-do-commands spec false statements)\n (reset! test-db-initialized? true)))\n ;; (println \"### Done with executing create-test-db! with details:\" details)\n))\n (f))\n\n#_{:clj-kondo/ignore [:warn-on-reflection]}\n(defn exec-statements\n ([statements details-map]\n (exec-statements statements details-map nil))\n ([statements details-map clickhouse-settings]\n (sql-jdbc.execute/do-with-connection-with-options\n :clickhouse\n (sql-jdbc.conn/connection-details->spec :clickhouse (merge {:engine :clickhouse} details-map))\n {:write? true}\n (fn [^java.sql.Connection conn]\n (doseq [statement statements]\n ;; (println \"Executing:\" statement)\n (let [^com.clickhouse.jdbc.ConnectionImpl clickhouse-conn (.unwrap conn com.clickhouse.jdbc.ConnectionImpl)\n query-settings (new com.clickhouse.client.api.query.QuerySettings)]\n (with-open [jdbcStmt (.createStatement conn)]\n (when clickhouse-settings\n (doseq [[k v] clickhouse-settings] (.setOption query-settings k v)))\n (.setDefaultQuerySettings clickhouse-conn query-settings)\n (.execute jdbcStmt statement))))))))\n\n(defn do-with-test-db\n \"Execute a test function using the test dataset\"\n [f]\n (t2.with-temp/with-temp\n [:model/Database database\n {:engine :clickhouse\n :details (tx/dbdef->connection-details :clickhouse :db {:database-name \"metabase_test\"})}]\n (sync/sync-db-metadata! database)\n (f database)))\n\n(defmethod tx/dataset-already-loaded? :clickhouse\n [driver dbdef]\n (let [tabledef (first (:table-definitions dbdef))\n db-name (ddl.i/format-name :clickhouse (:database-name dbdef))\n table-name (ddl.i/format-name :clickhouse (:table-name tabledef))\n details (tx/dbdef->connection-details :clickhouse :db {:database-name db-name})]\n (sql-jdbc.execute/do-with-connection-with-options\n driver\n (sql-jdbc.conn/connection-details->spec driver details)\n {:write? false}\n (fn [^java.sql.Connection conn]\n (with-open [rset (.getTables (.getMetaData conn)\n #_catalog nil\n #_schema-pattern db-name\n #_table-pattern table-name\n #_types (into-array String [\"TABLE\"]))]\n ;; if the ResultSet returns anything we know the table is already loaded.\n (.next rset))))))\n" }, { "path": "test/metabase/test/data/clickhouse_datasets.sql", "content": "DROP DATABASE IF EXISTS `metabase_test`;\nCREATE DATABASE `metabase_test`;\n\nCREATE TABLE `metabase_test`.`metabase_test_lowercases`\n(\n id UInt8,\n mystring Nullable(String)\n) ENGINE = Memory;\n\nINSERT INTO `metabase_test`.`metabase_test_lowercases`\nVALUES (1, 'Я_1'), (2, 'R'), (3, 'Я_2'), (4, 'Я'), (5, 'я'), (6, NULL);\n\nCREATE TABLE `metabase_test`.`enums_test`\n(\n enum1 Enum8('foo' = 0, 'bar' = 1, 'foo bar' = 2),\n enum2 Enum16('click' = 0, 'house' = 1),\n enum3 Enum8('qaz' = 42, 'qux' = 23)\n) ENGINE = Memory;\n\nINSERT INTO `metabase_test`.`enums_test` (enum1, enum2, enum3)\nVALUES ('foo', 'house', 'qaz'),\n ('foo bar', 'click', 'qux'),\n ('bar', 'house', 'qaz');\n\nCREATE TABLE `metabase_test`.`ipaddress_test`\n(\n ipvfour Nullable(IPv4),\n ipvsix Nullable(IPv6)\n) Engine = Memory;\n\nINSERT INTO `metabase_test`.`ipaddress_test` (ipvfour, ipvsix)\nVALUES (toIPv4('127.0.0.1'), toIPv6('0:0:0:0:0:0:0:1')),\n (toIPv4('0.0.0.0'), toIPv6('2001:438:ffff:0:0:0:407d:1bc1')),\n (null, null);\n\nCREATE TABLE `metabase_test`.`boolean_test`\n(\n ID Int32,\n b1 Bool,\n b2 Nullable(Bool)\n) ENGINE = Memory;\n\nINSERT INTO `metabase_test`.`boolean_test` (ID, b1, b2)\nVALUES (1, true, true),\n (2, false, true),\n (3, true, false);\n\nCREATE TABLE `metabase_test`.`maps_test`\n(\n m Map(String, UInt64)\n) ENGINE = Memory;\n\nINSERT INTO `metabase_test`.`maps_test`\nVALUES ({'key1':1,'key2':10}),\n ({'key1':2,'key2':20}),\n ({'key1':3,'key2':30});\n\n\nCREATE TABLE `metabase_test`.`array_of_tuples_test`\n(\n t Array(Tuple(String, UInt32))\n) Engine = Memory;\n\nINSERT INTO `metabase_test`.`array_of_tuples_test` (t)\nVALUES ([('foobar', 1234), ('qaz', 0)]),\n ([]);\n\n-- Used for testing that AggregateFunction columns are excluded,\n-- while SimpleAggregateFunction columns are preserved\nCREATE TABLE `metabase_test`.`aggregate_functions_filter_test`\n(\n idx UInt8,\n a AggregateFunction(uniq, String),\n lowest_value SimpleAggregateFunction(min, UInt8),\n count SimpleAggregateFunction(sum, Int64)\n) ENGINE Memory;\n\nINSERT INTO `metabase_test`.`aggregate_functions_filter_test`\n (idx, lowest_value, count)\nVALUES (42, 144, 255255);\n\n-- Materialized views (testing .inner tables exclusion)\nCREATE TABLE `metabase_test`.`wikistat`\n(\n `date` Date,\n `project` LowCardinality(String),\n `hits` UInt32\n) ENGINE = Memory;\n\nCREATE MATERIALIZED VIEW `metabase_test`.`wikistat_mv` ENGINE =Memory AS\nSELECT date, project, sum(hits) AS hits\nFROM `metabase_test`.`wikistat`\nGROUP BY date, project;\n\nINSERT INTO `metabase_test`.`wikistat`\nVALUES (now(), 'foo', 10),\n (now(), 'bar', 10),\n (now(), 'bar', 20);\n\n-- Used in sum-where tests\nCREATE TABLE `metabase_test`.`sum_if_test_int`\n(\n id Int64,\n int_value Int64,\n discriminator String\n) ENGINE = Memory;\n\nINSERT INTO `metabase_test`.`sum_if_test_int`\nVALUES (1, 1, 'foo'),\n (2, 1, 'foo'),\n (3, 3, 'bar'),\n (4, 5, 'bar');\n\nCREATE TABLE `metabase_test`.`sum_if_test_float`\n(\n id Int64,\n float_value Float64,\n discriminator String\n) ENGINE = Memory;\n\nINSERT INTO `metabase_test`.`sum_if_test_float`\nVALUES (1, 1.1, 'foo'),\n (2, 1.44, 'foo'),\n (3, 3.5, 'bar'),\n (4, 5.77, 'bar');\n\n-- Temporal bucketing tests\nCREATE TABLE `metabase_test`.`temporal_bucketing_server_tz`\n(\n start_of_year DateTime,\n mid_of_year DateTime,\n end_of_year DateTime\n) ENGINE = Memory;\n\nINSERT INTO `metabase_test`.`temporal_bucketing_server_tz`\nVALUES ('2022-01-01 00:00:00',\n '2022-06-20 06:32:54',\n '2022-12-31 23:59:59');\n\nCREATE TABLE `metabase_test`.`temporal_bucketing_column_tz`\n(\n start_of_year DateTime('America/Los_Angeles'),\n mid_of_year DateTime('America/Los_Angeles'),\n end_of_year DateTime('America/Los_Angeles')\n) ENGINE = Memory;\n\nINSERT INTO `metabase_test`.`temporal_bucketing_column_tz`\nVALUES (toDateTime('2022-01-01 00:00:00', 'America/Los_Angeles'),\n toDateTime('2022-06-20 06:32:54', 'America/Los_Angeles'),\n toDateTime('2022-12-31 23:59:59', 'America/Los_Angeles'));\n\nCREATE TABLE `metabase_test`.`datetime_diff_nullable` (\n idx Int32,\n dt64 Nullable(DateTime64(3, 'UTC')),\n dt Nullable(DateTime('UTC')),\n d Nullable(Date)\n) ENGINE Memory;\n\nINSERT INTO `metabase_test`.`datetime_diff_nullable`\nVALUES (42, '2022-01-01 00:00:00.000', '2022-06-20 06:32:54', '2022-07-22'),\n (43, '2022-01-01 00:00:00.000', NULL, NULL),\n (44, NULL, '2022-06-20 06:32:54', '2022-07-22'),\n (45, NULL, NULL, NULL);\n\nDROP DATABASE IF EXISTS `metabase_db_scan_test`;\nCREATE DATABASE `metabase_db_scan_test`;\n\nCREATE TABLE `metabase_db_scan_test`.`table1` (i Int32) ENGINE = Memory;\nCREATE TABLE `metabase_db_scan_test`.`table2` (i Int64) ENGINE = Memory;\n\n-- Base type matching tests\nCREATE TABLE `metabase_test`.`enums_base_types` (\n c1 Nullable(Enum8('America/New_York')),\n c2 Enum8('BASE TABLE' = 1, 'VIEW' = 2, 'FOREIGN TABLE' = 3, 'LOCAL TEMPORARY' = 4, 'SYSTEM VIEW' = 5),\n c3 Enum8('NO', 'YES'),\n c4 Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2),\n c5 Nullable(Enum8('GLOBAL' = 0, 'DATABASE' = 1, 'TABLE' = 2)),\n c6 Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2))\n) ENGINE Memory;\nCREATE TABLE `metabase_test`.`date_base_types` (\n c1 Date,\n c2 Date32,\n c3 Nullable(Date),\n c4 Nullable(Date32)\n) ENGINE Memory;\nCREATE TABLE `metabase_test`.`datetime_base_types` (\n c1 Nullable(DateTime('America/New_York')),\n c2 DateTime('America/New_York'),\n c3 DateTime,\n c4 DateTime64(3),\n c5 DateTime64(9, 'America/New_York'),\n c6 Nullable(DateTime64(6, 'America/New_York')),\n c7 Nullable(DateTime64(0)),\n c8 Nullable(DateTime)\n) ENGINE Memory;\nCREATE TABLE `metabase_test`.`integer_base_types` (\n c1 UInt8,\n c2 UInt16,\n c3 UInt32,\n c4 UInt64,\n c5 UInt128,\n c6 UInt256,\n c7 Int8,\n c8 Int16,\n c9 Int32,\n c10 Int64,\n c11 Int128,\n c12 Int256,\n c13 Nullable(Int32)\n) ENGINE Memory;\nCREATE TABLE `metabase_test`.`numeric_base_types` (\n c1 Float32,\n c2 Float64,\n c3 Decimal(4, 2),\n c4 Decimal32(7),\n c5 Decimal64(12),\n c6 Decimal128(24),\n c7 Decimal256(42),\n c8 Nullable(Float32),\n c9 Nullable(Decimal(4, 2)),\n c10 Nullable(Decimal256(42))\n) ENGINE Memory;\nCREATE TABLE `metabase_test`.`string_base_types` (\n c1 String,\n c2 LowCardinality(String),\n c3 FixedString(32),\n c4 Nullable(String),\n c5 LowCardinality(FixedString(4))\n) ENGINE Memory;\nCREATE TABLE `metabase_test`.`misc_base_types` (\n c1 Boolean,\n c2 UUID,\n c3 IPv4,\n c4 IPv6,\n c5 Map(Int32, String),\n c6 Nullable(Boolean),\n c7 Nullable(UUID),\n c8 Nullable(IPv4),\n c9 Nullable(IPv6),\n c10 Tuple(String, Int32)\n) ENGINE Memory;\nCREATE TABLE `metabase_test`.`array_base_types` (\n c1 Array(String),\n c2 Array(Nullable(Int32)),\n c3 Array(Array(LowCardinality(FixedString(32)))),\n c4 Array(Array(Array(String)))\n) ENGINE Memory;\nCREATE TABLE `metabase_test`.`low_cardinality_nullable_base_types` (\n c1 LowCardinality(Nullable(String)),\n c2 LowCardinality(Nullable(FixedString(16)))\n) ENGINE Memory;\n\n-- can-connect tests (odd database names)\nDROP DATABASE IF EXISTS `Special@Characters~`;\nCREATE DATABASE `Special@Characters~`;\n\n-- arrays inner types test\nCREATE TABLE `metabase_test`.`arrays_inner_types`\n(\n `arr_str` Array(String),\n `arr_nstr` Array(Nullable(String)),\n `arr_dec` Array(Decimal(18, 4)),\n `arr_ndec` Array(Nullable(Decimal(18, 4)))\n)\nENGINE Memory;\nINSERT INTO `metabase_test`.`arrays_inner_types` VALUES (\n ['a', 'b', 'c'],\n [NULL, 'd', 'e'],\n [1, 2, 3],\n [4, NULL, 5]\n);\n\nCREATE TABLE `metabase_test`.`unsigned_int_types`\n(\n `u8` UInt8,\n `u16` UInt16,\n `u32` UInt32,\n `u64` UInt64\n) ENGINE Memory;\nINSERT INTO `metabase_test`.`unsigned_int_types`\nVALUES (255, 65535, 4294967295, 18446744073709551615);\n\nCREATE TABLE `metabase_test`.`fixed_strings`\n(\n `f1` FixedString(4),\n `f2` LowCardinality(FixedString(4)),\n `f3` Nullable(FixedString(4)),\n `f4` LowCardinality(Nullable(FixedString(4)))\n) ENGINE Memory;\nINSERT INTO `metabase_test`.`fixed_strings`\nVALUES ('val1', 'val2', 'val3', 'val4');\n" } ]