[ { "path": ".editorconfig", "content": "root = true\n\n[*]\ntrim_trailing_whitespace = true\ninsert_final_newline = true\n\n[*.{py,pyx,pxd,pxi,h}]\nindent_size = 4\nindent_style = space\n\n[*.yml]\nindent_size = 2\nindent_style = space\n\n[edb_stat_statements/*.{c,h,l,y,pl,pm}]\nindent_style = tab\nindent_size = tab\ntab_width = 4\n" }, { "path": ".github/ISSUE_TEMPLATE/bug_report.md", "content": "---\nname: Bug report\nabout: Create a report to help us improve\n---\n\n\n\n\n\n- Gel Version:\n- Gel CLI Version:\n- OS Version:\n\nSteps to Reproduce:\n\n1.\n2.\n\n\n\nSchema:\n" }, { "path": ".github/ISSUE_TEMPLATE/config.yml", "content": "blank_issues_enabled: false\ncontact_links:\n - name: Problem with Gel UI\n url: https://github.com/geldata/gel-ui/issues\n about: If you've found a bug or have a feature request for Gel UI, please open an issue in the gel-ui repo.\n - name: Long question or idea\n url: https://github.com/geldata/gel/discussions\n about: Ask long-form questions and discuss ideas.\n - name: Quick questions or chat\n url: https://www.geldata.com/p/discord\n about: Ask quick questions or simply chat on the Gel Discord server.\n" }, { "path": ".github/ISSUE_TEMPLATE/feature_request.md", "content": "---\nname: Feature request\nabout: Suggest a feature for Gel\n\n---\n\n\n\n\n" }, { "path": ".github/Makefile", "content": ".PHONY: all\n\nROOT = $(dir $(realpath $(firstword $(MAKEFILE_LIST))))\n\nall: \\\n\tworkflows/build.nightly.yml \\\n\tworkflows/build.release.yml \\\n\tworkflows/build.testing.yml \\\n\tworkflows/build.dryrun.yml \\\n\tworkflows/build.ls-nightly.yml \\\n\tworkflows/tests.yml \\\n\tworkflows/tests.pool.yml \\\n\tworkflows/tests.managed-pg.yml \\\n\tworkflows/tests.ha.yml \\\n\tworkflows/tests.pg-versions.yml \\\n\tworkflows/tests.patches.yml \\\n\tworkflows/tests.inplace.yml \\\n\tworkflows/tests.inplace7x.yml \\\n\tworkflows/tests.reflection.yml \\\n\nworkflows/build.%.yml: workflows.src/build.%.tpl.yml workflows.src/build.%.targets.yml workflows.src/build.inc.yml\n\t$(ROOT)/workflows.src/render.py --workflow=build build.$* build.$*.targets.yml\n\nworkflows/tests.yml: workflows.src/tests.tpl.yml workflows.src/tests.targets.yml workflows.src/tests.inc.yml\n\t$(ROOT)/workflows.src/render.py --workflow=test tests tests.targets.yml\n\nworkflows/tests.%.yml: workflows.src/tests.%.tpl.yml workflows.src/tests.%.targets.yml workflows.src/tests.inc.yml\n\t$(ROOT)/workflows.src/render.py --workflow=test tests.$* tests.$*.targets.yml\n" }, { "path": ".github/aws-aurora/.gitignore", "content": "/.terraform/\n/terraform.tfstate*\n/.terraform.lock.hcl\n/.terraform.tfstate.lock.info\n" }, { "path": ".github/aws-aurora/main.tf", "content": "variable \"vpc_id\" {\n description = \"VPC ID\"\n}\n\nvariable \"sg_id\" {\n description = \"security group ID\"\n}\n\nvariable \"password\" {\n description = \"password, provide through your ENV variables\"\n}\n\nmodule \"aurora\" {\n source = \"terraform-aws-modules/rds-aurora/aws\"\n version = \"~> 5.0\"\n\n name = \"aws-aurora-instance\"\n engine = \"aurora-postgresql\"\n engine_version = \"13.4\"\n instance_type = \"db.r6g.large\"\n\n\n vpc_id = var.vpc_id\n db_subnet_group_name\t= \"default\"\n\n replica_count = 1\n create_security_group = false\n vpc_security_group_ids = [var.sg_id]\n\n\n storage_encrypted = true\n apply_immediately = true\n\n username = \"edbtest\"\n password = var.password\n create_random_password = false\n\n enabled_cloudwatch_logs_exports = [\"postgresql\"]\n publicly_accessible = true\n skip_final_snapshot = true\n\n tags = {\n Environment = \"dev\"\n Terraform = \"true\"\n }\n}\n\noutput \"rds_cluster_endpoint\" {\n description = \"The cluster endpoint\"\n value = module.aurora.rds_cluster_endpoint\n}\n" }, { "path": ".github/aws-rds/.gitignore", "content": "/.terraform/\n/terraform.tfstate*\n" }, { "path": ".github/aws-rds/.terraform.lock.hcl", "content": "# This file is maintained automatically by \"terraform init\".\n# Manual edits may be lost in future updates.\n\nprovider \"registry.terraform.io/hashicorp/aws\" {\n version = \"3.31.0\"\n hashes = [\n \"h1:Wou3ZnO10ZvN+n1iwyuaxn3zyGMFj9KYL+9IFb0gGkw=\",\n \"zh:07f5b2f4cfaa25e26a4062ac675e3e5aaf65bb21b94b8fd7f30d576398e7410f\",\n \"zh:08a2154ad29ae130ea9e46948b7b332ec4b45321b4852b45ba60adcfd049f8d6\",\n \"zh:35ed643c2b999021ad56b49f7d9d3a77c98d152477fe54b5c8a68f696bb1a0b7\",\n \"zh:3a8dc51b4be1c04130fd76cda4280019020b276336d307e7074ad52f35d4fdda\",\n \"zh:3c910c4f25e3ffd6d84f051c32161f03d1843753cd545e769757d7b42d654003\",\n \"zh:5d23f316f89937cbda36207271bbe150f633298f96d4644fd02063fc6bf0c28f\",\n \"zh:61fedb2915c5188c6550677a10acb955f32834bbe99ba0cafb2a118be282827b\",\n \"zh:65076a6899c0781ce95064d47d587ad07f80becd1510e4c475e4554131caec09\",\n \"zh:acca833c2d9985e46298323222285b370ea7cf5299b131dbdfc7c3e66fa32401\",\n \"zh:c212cf8ba7fdf64e75accf7e745f76d2349b00553ebd928cc6cafbfda99d97b7\",\n \"zh:cd3f5e89ac5f5cf3f8fed3aca4cc50261d537b60a3490feaddf9ba2f06e5e7aa\",\n ]\n}\n" }, { "path": ".github/aws-rds/main.tf", "content": "resource \"aws_db_instance\" \"default\" {\n allocated_storage = 10\n engine = \"postgres\"\n engine_version = \"13.4\"\n instance_class = \"db.m6g.large\"\n name = \"edbtest\"\n username = \"edbtest\"\n password = var.password\n parameter_group_name = \"default.postgres13\"\n skip_final_snapshot = true\n auto_minor_version_upgrade = false\n publicly_accessible = true\n vpc_security_group_ids = [var.sg_id]\n}\n" }, { "path": ".github/aws-rds/outputs.tf", "content": "output \"db_instance_id\" {\n value = aws_db_instance.default.id\n}\n\noutput \"db_instance_address\" {\n value = aws_db_instance.default.address\n}\n" }, { "path": ".github/aws-rds/variables.tf", "content": "variable \"sg_id\" {\n description = \"security group ID\"\n}\n\nvariable \"password\" {\n description = \"password, provide through your ENV variables\"\n}\n" }, { "path": ".github/do-database/.gitignore", "content": "/.terraform/\n/terraform.tfstate*\n" }, { "path": ".github/do-database/.terraform.lock.hcl", "content": "# This file is maintained automatically by \"terraform init\".\n# Manual edits may be lost in future updates.\n\nprovider \"registry.terraform.io/digitalocean/digitalocean\" {\n version = \"2.6.0\"\n constraints = \"2.6.0\"\n hashes = [\n \"h1:P1C7e6RlhLpi6KuE/sMruDdM5zZisJwMuKGbnxg8tAw=\",\n \"zh:088c2a4eb9579947d50d8bcd722e75f2f1839acae302c8d43133b1da9926dae3\",\n \"zh:323ba833d011371ca6d953752b133c0acad6462176cd2f804077a5f9d892cd2e\",\n \"zh:3fbc64f1fabe57b6df49511c0d8753f1bbf776d5824ba060a51961d2a4265097\",\n \"zh:4c90a933e23288ee2db2228e4e30055882d91bed831c2191cbecd849b27e44cb\",\n \"zh:62f1cf4c82e5fcaf1a17e39cb96638f006b303758813a6c5ecb08bc93cd93364\",\n \"zh:68ad1354e9f925477dc41e658e84a4996ba662920bbc61a2680235b94811169b\",\n \"zh:9119b573c59429c2dfacb7d95b39c4e021783b8281ecd68f1621ad4a17c112cd\",\n \"zh:9c15e3660f2399c25ee3ad53bd54927a6529d1393a54f1e1c2a523e0369dea46\",\n \"zh:bc88f68bf6a6b5e803734f06731e31d61a5977ed1a638bfe102a54094c4d4030\",\n \"zh:c2b013a5d7e60b31211b0f8c0dd898840b8f1aa7225318da05def33b5edb9388\",\n \"zh:e46e21f6ffa7aac11ade8ab4b87a28ac405ef40a35793cef1f1fd6db6d8e5a0a\",\n \"zh:e879643369e03abc192fbcf7ab06611bb8f36d37ceb5641ba05d58869f10ab7c\",\n \"zh:ee9b56400e545ce1805842b795179a004313b8a947bd8f3490f5c5a0cb7703e5\",\n \"zh:fb44861ae0b58b594aa4e565e0ed06bce939753b14a20b4abd3e8276e839e7a7\",\n ]\n}\n" }, { "path": ".github/do-database/main.tf", "content": "terraform {\n required_providers {\n digitalocean = {\n source = \"digitalocean/digitalocean\"\n version = \"2.6.0\"\n }\n }\n}\n\nvariable \"do_token\" {}\n\nprovider \"digitalocean\" {\n\n token = var.do_token\n}\n\nresource \"digitalocean_database_cluster\" \"default\" {\n name = \"edbtest\"\n engine = \"pg\"\n version = \"13\"\n size = \"db-s-4vcpu-8gb\"\n region = \"nyc1\"\n node_count = 1\n}\n" }, { "path": ".github/do-database/outputs.tf", "content": "output \"db_instance_address\" {\n value = digitalocean_database_cluster.default.host\n}\n\noutput \"db_instance_port\" {\n value = digitalocean_database_cluster.default.port\n}\n\noutput \"db_instance_user\" {\n value = digitalocean_database_cluster.default.user\n}\n\noutput \"db_instance_password\" {\n value = digitalocean_database_cluster.default.password\n sensitive = true\n}\n\noutput \"db_instance_database\" {\n value = digitalocean_database_cluster.default.database\n}\n" }, { "path": ".github/gcp-cloud-sql/.gitignore", "content": "/.terraform/\n/terraform.tfstate*\n" }, { "path": ".github/gcp-cloud-sql/.terraform.lock.hcl", "content": "# This file is maintained automatically by \"terraform init\".\n# Manual edits may be lost in future updates.\n\nprovider \"registry.terraform.io/hashicorp/google\" {\n version = \"3.62.0\"\n hashes = [\n \"h1:FgfQz6EhKglcoU7vu1srYqEQFXy1Dti9MoZCxW8HL/w=\",\n \"zh:26e44482924c9d22624054dcebf23c89b102aee6b5c66675747cf2f7274cf703\",\n \"zh:518ebd73eb8f286f60a0c74970cd4e06883962c4af57f2899bc790d89e04038f\",\n \"zh:814036d49d5034cf26fd2239fc57075b42982e1f76ab703fa1cd7609802d979f\",\n \"zh:822dce72d1a77e1418b0e9187b4fe6f3e47b38ea5e51b81e5912074a8be3a7b7\",\n \"zh:981fc6780e1e9c756390727b94ebd822490f7504a05a26c818922da5635ff9b8\",\n \"zh:9a1a7e76ac6c37922261bdb148052fcdcbaf1f521ade68e26b430c106f1974b1\",\n \"zh:cb67b6abed58b6d1b789a72690154fcf35707f65c3fca1936bf72c0c819a03dd\",\n \"zh:cb87e8425b0eb97d80627243a37a67f0f81640499416ad32f1b786cc9d78c6f4\",\n \"zh:d3754c3f05dc9bbd4933b45676144c2dd456de775bff0252c058e0cff94b8f21\",\n \"zh:e2d8b0a78d698e92035e339782b299108d6021768ea4d97d150106c524f84ca1\",\n ]\n}\n" }, { "path": ".github/gcp-cloud-sql/main.tf", "content": "variable \"password\" {}\n\nprovider \"google\" {\n region = \"us-east1\"\n}\n\nresource \"google_sql_database_instance\" \"default\" {\n database_version = \"POSTGRES_13\"\n deletion_protection = false\n\n settings {\n tier = \"db-custom-1-3840\"\n\n ip_configuration {\n authorized_networks {\n value = \"0.0.0.0/0\"\n }\n }\n }\n}\n\nresource \"google_sql_user\" \"users\" {\n instance = google_sql_database_instance.default.name\n name = \"postgres\"\n password = var.password\n deletion_policy = \"ABANDON\"\n}\n\noutput \"db_instance_address\" {\n value = google_sql_database_instance.default.public_ip_address\n}\n" }, { "path": ".github/heroku-postgres/.gitignore", "content": "/.terraform/\n/terraform.tfstate*\n/.terraform.lock.hcl\n/.terraform.tfstate.lock.info\n" }, { "path": ".github/heroku-postgres/main.tf", "content": "terraform {\n required_providers {\n heroku = {\n source = \"heroku/heroku\"\n version = \"~> 4.0\"\n }\n }\n}\n\nresource \"heroku_addon\" \"database\" {\n app = \"edgedb-heroku-ci\"\n plan = \"heroku-postgresql:mini\"\n config = {\n version = \"14\"\n }\n}\n\noutput \"heroku_postgres_dsn\" {\n value = heroku_addon.database.config_var_values.DATABASE_URL\n sensitive = true\n}\n" }, { "path": ".github/scripts/docs/preview-deploy.js", "content": "const DOCS_SITE_REPO = {\n org: \"edgedb\",\n repo: \"edgedb.com\",\n ref: \"master\",\n};\n\nmodule.exports = async ({ github, context }) => {\n const { VERCEL_TOKEN, VERCEL_TEAM_ID } = process.env;\n\n if (!VERCEL_TOKEN || !VERCEL_TEAM_ID) {\n throw new Error(\n `cannot run docs preview deploy workflow, ` +\n `VERCEL_TOKEN or VERCEL_TEAM_ID secrets are missing`\n );\n }\n\n const prBranch = context.payload.pull_request.head.ref;\n const commitSHA = context.payload.pull_request.head.sha;\n const shortCommitSHA = commitSHA.slice(0, 8);\n\n const existingComments = (\n await github.rest.issues.listComments({\n owner: context.repo.owner,\n repo: context.repo.repo,\n issue_number: context.issue.number,\n })\n ).data;\n\n const commentHeader = `### Docs preview deploy\\n`;\n let commentMessage = commentHeader;\n\n let updateComment = existingComments.find(\n (c) =>\n c.performed_via_github_app?.slug === \"github-actions\" &&\n c.body?.startsWith(commentHeader)\n );\n\n let deploymentError = null;\n let deployment;\n try {\n deployment = await vercelFetch(\"https://api.vercel.com/v13/deployments\", {\n name: \"edgedb-docs\",\n gitSource: {\n type: \"github\",\n ...DOCS_SITE_REPO,\n },\n projectSettings: {\n buildCommand: `EDGEDB_REPO_BRANCH=${prBranch} EDGEDB_REPO_SHA=${commitSHA} yarn vercel-build`,\n },\n });\n\n commentMessage += `\\nšŸ”„ Deploying docs preview for commit ${shortCommitSHA}:\\n\\n`;\n } catch (e) {\n deploymentError = e;\n commentMessage += `\\nāŒ Failed to deploy docs preview for commit ${shortCommitSHA}:\\n\\n\\`\\`\\`\\n${e.message}\\n\\`\\`\\``;\n }\n\n commentMessage += `\\n\\n(Last updated: ${formatDatetime(new Date())})`;\n\n if (updateComment) {\n await github.rest.issues.updateComment({\n owner: context.repo.owner,\n repo: context.repo.repo,\n comment_id: updateComment.id,\n body: commentMessage,\n });\n } else {\n updateComment = (\n await github.rest.issues.createComment({\n owner: context.repo.owner,\n repo: context.repo.repo,\n issue_number: context.issue.number,\n body: commentMessage,\n })\n ).data;\n }\n\n if (deploymentError) {\n throw new Error(`Docs preview deployment failed: ${e.message}`);\n }\n\n let i = 0;\n while (i < 40) {\n await sleep(15_000);\n i++;\n\n const status = (\n await vercelFetch(\n `https://api.vercel.com/v13/deployments/${deployment.id}`\n )\n ).status;\n\n const latestComment = await github.rest.issues.getComment({\n owner: context.repo.owner,\n repo: context.repo.repo,\n comment_id: updateComment.id,\n });\n\n if (!latestComment.data.body.includes(shortCommitSHA)) {\n console.log(\"Skipping further updates, new deployment has started\");\n return;\n }\n\n if (status === \"READY\" || status === \"ERROR\" || status === \"CANCELED\") {\n await github.rest.issues.updateComment({\n owner: context.repo.owner,\n repo: context.repo.repo,\n comment_id: updateComment.id,\n body: `${commentHeader}${\n status === \"READY\"\n ? `\\nāœ… Successfully deployed docs preview for commit ${shortCommitSHA}:`\n : `\\nāŒ Docs preview deployment ${\n status === \"CANCELED\" ? \"was canceled\" : \"failed\"\n } for commit ${shortCommitSHA}:`\n }\\n\\n\\n\\n(Last updated: ${formatDatetime(\n new Date()\n )})`,\n });\n if (status !== \"READY\") {\n throw new Error(\n `Docs preview deployment failed with status ${status}: https://${deployment.url}`\n );\n }\n return;\n }\n }\n\n await github.rest.issues.updateComment({\n owner: context.repo.owner,\n repo: context.repo.repo,\n comment_id: updateComment.id,\n body: `${commentHeader}\nāŒ Timed out waiting for deployment status to succeed or fail for commit ${shortCommitSHA}:\\n\\n\\n\\n(Last updated: ${formatDatetime(new Date())})`,\n });\n throw new Error(\"Timed out waiting for deployment status to succeed or fail\");\n};\n\nasync function vercelFetch(url, body) {\n const { VERCEL_TOKEN, VERCEL_TEAM_ID } = process.env;\n const _url = new URL(url);\n url = `${_url.origin}${_url.pathname}?${new URLSearchParams({\n teamId: VERCEL_TEAM_ID,\n })}`;\n\n let res;\n try {\n res = await fetch(url, {\n body: body ? JSON.stringify(body) : undefined,\n headers: {\n Authorization: `Bearer ${VERCEL_TOKEN}`,\n \"Content-Type\": body ? \"application/json\" : undefined,\n },\n method: body ? \"post\" : \"get\",\n });\n } catch (e) {\n throw new Error(`vercel api request failed: ${e}`);\n }\n\n if (res.ok) {\n return await res.json();\n } else {\n let body;\n try {\n body = await res.text();\n } catch (e) {\n // ignore\n }\n throw new Error(\n `vercel api request failed: ${res.status} ${res.statusText}, ${body}`\n );\n }\n}\n\nfunction formatDatetime(date) {\n return date.toLocaleString(\"en-US\", {\n year: \"numeric\",\n month: \"short\",\n day: \"numeric\",\n hour: \"numeric\",\n minute: \"numeric\",\n second: \"numeric\",\n hourCycle: \"h24\",\n timeZoneName: \"short\",\n });\n}\n\nfunction sleep(milliseconds) {\n return new Promise((resolve) => setTimeout(resolve, milliseconds));\n}\n" }, { "path": ".github/scripts/patches/compute-ipu-versions.py", "content": "# Compute prior minor versions to test upgrading from\n\n\nimport json\nimport os\nimport pathlib\nimport re\nimport sys\nfrom urllib import request\n\nsys.path.append(str(pathlib.Path(__file__).parent.parent.parent.parent))\n\nimport edb.buildmeta\n\nbase = 'https://packages.geldata.com'\nu = f'{base}/archive/.jsonindexes/x86_64-unknown-linux-gnu.json'\ndata = json.loads(request.urlopen(u).read())\n\nu = f'{base}/archive/.jsonindexes/x86_64-unknown-linux-gnu.testing.json'\ndata_testing = json.loads(request.urlopen(u).read())\n\nversion = edb.buildmeta.EDGEDB_MAJOR_VERSION - 1\n\nversions = []\nprerelease_versions = []\nfor obj in data['packages'] + data_testing['packages']:\n if (\n obj['basename'] == 'gel-server'\n and obj['version_details']['major'] == version\n and (\n not obj['version_details']['prerelease']\n or obj['version_details']['prerelease'][0]['phase'] in ('beta', 'rc')\n )\n ):\n l = (\n versions if not obj['version_details']['prerelease']\n else prerelease_versions\n )\n l.append((\n obj['version'],\n obj['basename'],\n base + obj['installrefs'][0]['ref'],\n ))\n\nprerelease_versions.sort(key=lambda x: x[0])\nif not versions:\n # Some 7.x prerelease versions are busted due to having taken\n # extension patches that we don't intend to bundle with 8.x.\n # Only look at the last.\n versions = prerelease_versions[-1:]\n\nversions.sort(key=lambda x: x[0])\nif len(versions) > 3:\n # We want to try 6.0 and 6.2\n versions = [versions[0], versions[2], versions[-1]]\nelif len(versions) > 1:\n versions = [versions[0], versions[-1]]\n\nmatrix = {\n \"include\": [\n {\"edgedb-version\": v, \"edgedb-url\": url, \"edgedb-basename\": base}\n for v, base, url in versions\n ]\n}\n\nprint(\"matrix:\", matrix)\nif output := os.getenv('GITHUB_OUTPUT'):\n with open(output, 'a') as f:\n print(f'matrix={json.dumps(matrix)}', file=f)\n" }, { "path": ".github/scripts/patches/compute-versions.py", "content": "# Compute prior minor versions to test upgrading from\n\nimport json\nimport os\nimport re\nfrom urllib import request\n\nbase = 'https://packages.edgedb.com'\nu = f'{base}/archive/.jsonindexes/x86_64-unknown-linux-gnu.json'\ndata = json.loads(request.urlopen(u).read())\n\nu = f'{base}/archive/.jsonindexes/x86_64-unknown-linux-gnu.testing.json'\ndata_testing = json.loads(request.urlopen(u).read())\n\n\nbranch = os.getenv('GITHUB_BASE_REF') or os.getenv('GITHUB_REF_NAME')\nprint(\"BRANCH\", branch)\nversion = int(re.findall(r'\\d+', branch)[0])\n\nversions = []\nfor obj in data['packages'] + data_testing['packages']:\n if (\n obj['basename'] in {'gel-server', 'edgedb-server'}\n and obj['version_details']['major'] == version\n and (\n not obj['version_details']['prerelease']\n or obj['version_details']['prerelease'][0]['phase'] in ('beta', 'rc')\n )\n ):\n versions.append((\n obj['version'],\n obj['basename'],\n base + obj['installrefs'][0]['ref'],\n ))\n\nmatrix = {\n \"include\": [\n {\"edgedb-version\": v, \"edgedb-url\": url, \"edgedb-basename\": base, \"make-dbs\": mk}\n for v, base, url in versions\n for mk in [True, False]\n ]\n}\n\nprint(\"matrix:\", matrix)\nif output := os.getenv('GITHUB_OUTPUT'):\n with open(output, 'a') as f:\n print(f'matrix={json.dumps(matrix)}', file=f)\n" }, { "path": ".github/scripts/patches/create-databases.py", "content": "# Create databases on the older edgedb version\n\nimport edgedb\nimport subprocess\nimport sys\n\ncmd = [\n sys.argv[1], '-D' 'test-dir',\n '--testmode', '--security', 'insecure_dev_mode', '--port', '10000',\n]\nproc = subprocess.Popen(cmd)\n\ntry:\n db = edgedb.create_client(\n host='localhost', port=10000, tls_security='insecure'\n )\n for name in [\n 'json', 'functions', 'expressions', 'casts', 'policies', 'vector',\n 'scope', 'httpextauth',\n ]:\n db.execute(f'create database {name};')\n\n # For the scope database, let's actually migrate to it. This\n # will test that the migrations can still work after the upgrade.\n db2 = edgedb.create_client(\n host='localhost', port=10000, tls_security='insecure', database='scope'\n )\n with open(\"tests/schemas/cards.esdl\") as f:\n body = f.read()\n db2.execute(f'''\n START MIGRATION TO {{\n module default {{\n {body}\n }}\n }};\n POPULATE MIGRATION;\n COMMIT MIGRATION;\n ''')\n\n # Put something in the query cache\n db2.query(r'''\n SELECT User {\n name,\n id\n }\n ORDER BY User.name;\n ''')\n\n db2.close()\n\n # Compile a query from the CLI.\n # (At one point, having a cached query with proto version 1 caused\n # trouble...)\n cli_base = [\n 'gel',\n 'query',\n '-H',\n 'localhost',\n '-P',\n '10000',\n '-b',\n 'json',\n '--tls-security',\n 'insecure',\n ]\n subprocess.run(\n [*cli_base, 'select 1+1'],\n check=True,\n )\n\n # For the httpextauth database, create the proper extensions, so\n # that patching of the auth extension in place can get tested.\n db2 = edgedb.create_client(\n host='localhost', port=10000, tls_security='insecure',\n database='httpextauth'\n )\n db2.execute(f'''\n create extension pgcrypto;\n create extension auth;\n ''')\n db2.close()\n\nfinally:\n proc.terminate()\n proc.wait()\n" }, { "path": ".github/scripts/patches/test-downgrade.py", "content": "# Test downgrading a database after an upgrade\n\nimport edgedb\nimport os\nimport subprocess\nimport json\n\nversion = os.getenv('EDGEDB_VERSION')\ncmd = [\n f'edgedb-server-{version}/bin/edgedb-server', '-D' 'test-dir',\n '--testmode', '--security', 'insecure_dev_mode', '--port', '10000',\n]\nproc = subprocess.Popen(cmd)\n\ndb = edgedb.create_client(\n host='localhost', port=10000, tls_security='insecure',\n database='policies',\n)\n\ntry:\n # Test that a basic query works\n res = json.loads(db.query_json('''\n select Issue { name, number, watchers: {name} }\n filter .number = \"1\"\n '''))\n expected = [{\n \"name\": \"Release EdgeDB\",\n \"number\": \"1\",\n \"watchers\": [{\"name\": \"Yury\"}],\n }]\n\n assert res == expected, res\nfinally:\n proc.terminate()\n proc.wait()\n" }, { "path": ".github/workflows/.gitattributes", "content": "*.yml linguist-generated=true\n" }, { "path": ".github/workflows/build.dryrun.yml", "content": "name: Package Build Dry Run\n\non:\n workflow_dispatch:\n inputs:\n gelpkg_ref:\n description: \"gel-pkg git ref used to build the packages\"\n default: \"master\"\n metapkg_ref:\n description: \"metapkg git ref used to build the packages\"\n default: \"master\"\n\njobs:\n prep:\n runs-on: ubuntu-latest\n\n outputs:\n\n if_debian_buster_x86_64: ${{ steps.scm.outputs.if_debian_buster_x86_64 }}\n\n if_debian_buster_aarch64: ${{ steps.scm.outputs.if_debian_buster_aarch64 }}\n\n if_debian_bullseye_x86_64: ${{ steps.scm.outputs.if_debian_bullseye_x86_64 }}\n\n if_debian_bullseye_aarch64: ${{ steps.scm.outputs.if_debian_bullseye_aarch64 }}\n\n if_debian_bookworm_x86_64: ${{ steps.scm.outputs.if_debian_bookworm_x86_64 }}\n\n if_debian_bookworm_aarch64: ${{ steps.scm.outputs.if_debian_bookworm_aarch64 }}\n\n if_ubuntu_focal_x86_64: ${{ steps.scm.outputs.if_ubuntu_focal_x86_64 }}\n\n if_ubuntu_focal_aarch64: ${{ steps.scm.outputs.if_ubuntu_focal_aarch64 }}\n\n if_ubuntu_jammy_x86_64: ${{ steps.scm.outputs.if_ubuntu_jammy_x86_64 }}\n\n if_ubuntu_jammy_aarch64: ${{ steps.scm.outputs.if_ubuntu_jammy_aarch64 }}\n\n if_ubuntu_noble_x86_64: ${{ steps.scm.outputs.if_ubuntu_noble_x86_64 }}\n\n if_ubuntu_noble_aarch64: ${{ steps.scm.outputs.if_ubuntu_noble_aarch64 }}\n\n if_centos_8_x86_64: ${{ steps.scm.outputs.if_centos_8_x86_64 }}\n\n if_centos_8_aarch64: ${{ steps.scm.outputs.if_centos_8_aarch64 }}\n\n if_rockylinux_9_x86_64: ${{ steps.scm.outputs.if_rockylinux_9_x86_64 }}\n\n if_rockylinux_9_aarch64: ${{ steps.scm.outputs.if_rockylinux_9_aarch64 }}\n\n if_linux_x86_64: ${{ steps.scm.outputs.if_linux_x86_64 }}\n\n if_linux_aarch64: ${{ steps.scm.outputs.if_linux_aarch64 }}\n\n if_linuxmusl_x86_64: ${{ steps.scm.outputs.if_linuxmusl_x86_64 }}\n\n if_linuxmusl_aarch64: ${{ steps.scm.outputs.if_linuxmusl_aarch64 }}\n\n if_macos_x86_64: ${{ steps.scm.outputs.if_macos_x86_64 }}\n\n if_macos_aarch64: ${{ steps.scm.outputs.if_macos_aarch64 }}\n\n\n steps:\n - uses: actions/checkout@v4\n\n\n - name: Determine SCM revision\n id: scm\n shell: bash\n run: |\n rev=$(git rev-parse HEAD)\n jq_filter='.packages[] | select(.basename == \"gel-server\") | select(.architecture == $ARCH) | .version_details.metadata.scm_revision | . as $rev | select(($rev != null) and ($REV | startswith($rev)))'\n\n key=\"debian-buster-x86_64\"\n val=true\n\n\n idx_file=buster.nightly.json\n url=https://packages.edgedb.com/apt/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"x86_64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"debian-buster-aarch64\"\n val=true\n\n\n idx_file=buster.nightly.json\n url=https://packages.edgedb.com/apt/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"aarch64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"debian-bullseye-x86_64\"\n val=true\n\n\n idx_file=bullseye.nightly.json\n url=https://packages.edgedb.com/apt/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"x86_64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"debian-bullseye-aarch64\"\n val=true\n\n\n idx_file=bullseye.nightly.json\n url=https://packages.edgedb.com/apt/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"aarch64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"debian-bookworm-x86_64\"\n val=true\n\n\n idx_file=bookworm.nightly.json\n url=https://packages.edgedb.com/apt/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"x86_64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"debian-bookworm-aarch64\"\n val=true\n\n\n idx_file=bookworm.nightly.json\n url=https://packages.edgedb.com/apt/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"aarch64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"ubuntu-focal-x86_64\"\n val=true\n\n\n idx_file=focal.nightly.json\n url=https://packages.edgedb.com/apt/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"x86_64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"ubuntu-focal-aarch64\"\n val=true\n\n\n idx_file=focal.nightly.json\n url=https://packages.edgedb.com/apt/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"aarch64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"ubuntu-jammy-x86_64\"\n val=true\n\n\n idx_file=jammy.nightly.json\n url=https://packages.edgedb.com/apt/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"x86_64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"ubuntu-jammy-aarch64\"\n val=true\n\n\n idx_file=jammy.nightly.json\n url=https://packages.edgedb.com/apt/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"aarch64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"ubuntu-noble-x86_64\"\n val=true\n\n\n idx_file=noble.nightly.json\n url=https://packages.edgedb.com/apt/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"x86_64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"ubuntu-noble-aarch64\"\n val=true\n\n\n idx_file=noble.nightly.json\n url=https://packages.edgedb.com/apt/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"aarch64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"centos-8-x86_64\"\n val=true\n\n\n idx_file=el8.nightly.json\n url=https://packages.edgedb.com/rpm/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"x86_64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"centos-8-aarch64\"\n val=true\n\n\n idx_file=el8.nightly.json\n url=https://packages.edgedb.com/rpm/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"aarch64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"rockylinux-9-x86_64\"\n val=true\n\n\n idx_file=el9.nightly.json\n url=https://packages.edgedb.com/rpm/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"x86_64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"rockylinux-9-aarch64\"\n val=true\n\n\n idx_file=el9.nightly.json\n url=https://packages.edgedb.com/rpm/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"aarch64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"linux-x86_64\"\n val=true\n\n\n idx_file=x86_64-unknown-linux-gnu.nightly.json\n url=https://packages.edgedb.com/archive/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"x86_64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"linux-aarch64\"\n val=true\n\n\n idx_file=aarch64-unknown-linux-gnu.nightly.json\n url=https://packages.edgedb.com/archive/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"aarch64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"linuxmusl-x86_64\"\n val=true\n\n\n idx_file=x86_64-unknown-linux-musl.nightly.json\n url=https://packages.edgedb.com/archive/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"x86_64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"linuxmusl-aarch64\"\n val=true\n\n\n idx_file=aarch64-unknown-linux-musl.nightly.json\n url=https://packages.edgedb.com/archive/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"aarch64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"macos-x86_64\"\n val=true\n\n\n idx_file=x86_64-unknown-linux-gnu.nightly.json\n url=https://packages.edgedb.com/archive/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"x86_64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"macos-aarch64\"\n val=true\n\n\n idx_file=aarch64-unknown-linux-gnu.nightly.json\n url=https://packages.edgedb.com/archive/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"aarch64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n\n\n build-debian-buster-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n if: needs.prep.outputs.if_debian_buster_x86_64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-debian-buster:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"buster\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-debian-buster-x86_64\n path: artifacts/debian-buster\n\n build-debian-buster-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n if: needs.prep.outputs.if_debian_buster_aarch64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-debian-buster:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"buster\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-debian-buster-aarch64\n path: artifacts/debian-buster\n\n build-debian-bullseye-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n if: needs.prep.outputs.if_debian_bullseye_x86_64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-debian-bullseye:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bullseye\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-debian-bullseye-x86_64\n path: artifacts/debian-bullseye\n\n build-debian-bullseye-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n if: needs.prep.outputs.if_debian_bullseye_aarch64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-debian-bullseye:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bullseye\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-debian-bullseye-aarch64\n path: artifacts/debian-bullseye\n\n build-debian-bookworm-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n if: needs.prep.outputs.if_debian_bookworm_x86_64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-debian-bookworm:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bookworm\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-debian-bookworm-x86_64\n path: artifacts/debian-bookworm\n\n build-debian-bookworm-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n if: needs.prep.outputs.if_debian_bookworm_aarch64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-debian-bookworm:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bookworm\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-debian-bookworm-aarch64\n path: artifacts/debian-bookworm\n\n build-ubuntu-focal-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n if: needs.prep.outputs.if_ubuntu_focal_x86_64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-ubuntu-focal:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"focal\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-ubuntu-focal-x86_64\n path: artifacts/ubuntu-focal\n\n build-ubuntu-focal-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n if: needs.prep.outputs.if_ubuntu_focal_aarch64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-ubuntu-focal:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"focal\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-ubuntu-focal-aarch64\n path: artifacts/ubuntu-focal\n\n build-ubuntu-jammy-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n if: needs.prep.outputs.if_ubuntu_jammy_x86_64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-ubuntu-jammy:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"jammy\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-ubuntu-jammy-x86_64\n path: artifacts/ubuntu-jammy\n\n build-ubuntu-jammy-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n if: needs.prep.outputs.if_ubuntu_jammy_aarch64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-ubuntu-jammy:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"jammy\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-ubuntu-jammy-aarch64\n path: artifacts/ubuntu-jammy\n\n build-ubuntu-noble-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n if: needs.prep.outputs.if_ubuntu_noble_x86_64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-ubuntu-noble:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"noble\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-ubuntu-noble-x86_64\n path: artifacts/ubuntu-noble\n\n build-ubuntu-noble-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n if: needs.prep.outputs.if_ubuntu_noble_aarch64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-ubuntu-noble:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"noble\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-ubuntu-noble-aarch64\n path: artifacts/ubuntu-noble\n\n build-centos-8-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n if: needs.prep.outputs.if_centos_8_x86_64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-centos-8:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"centos\"\n PKG_PLATFORM_VERSION: \"8\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-centos-8-x86_64\n path: artifacts/centos-8\n\n build-centos-8-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n if: needs.prep.outputs.if_centos_8_aarch64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-centos-8:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"centos\"\n PKG_PLATFORM_VERSION: \"8\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-centos-8-aarch64\n path: artifacts/centos-8\n\n build-rockylinux-9-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n if: needs.prep.outputs.if_rockylinux_9_x86_64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-rockylinux-9:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"rockylinux\"\n PKG_PLATFORM_VERSION: \"9\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-rockylinux-9-x86_64\n path: artifacts/rockylinux-9\n\n build-rockylinux-9-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n if: needs.prep.outputs.if_rockylinux_9_aarch64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-rockylinux-9:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"rockylinux\"\n PKG_PLATFORM_VERSION: \"9\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-rockylinux-9-aarch64\n path: artifacts/rockylinux-9\n\n build-linux-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n if: needs.prep.outputs.if_linux_x86_64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-linux-x86_64:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_GENERIC: true\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-linux-x86_64\n path: artifacts/linux-x86_64\n\n build-linux-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n if: needs.prep.outputs.if_linux_aarch64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-linux-aarch64:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_GENERIC: true\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-linux-aarch64\n path: artifacts/linux-aarch64\n\n build-linuxmusl-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n if: needs.prep.outputs.if_linuxmusl_x86_64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-linuxmusl-x86_64:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_GENERIC: true\n PKG_PLATFORM_LIBC: \"musl\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-linuxmusl-x86_64\n path: artifacts/linuxmusl-x86_64\n\n build-linuxmusl-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n if: needs.prep.outputs.if_linuxmusl_aarch64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-linuxmusl-aarch64:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_GENERIC: true\n PKG_PLATFORM_LIBC: \"musl\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-linuxmusl-aarch64\n path: artifacts/linuxmusl-aarch64\n\n build-macos-x86_64:\n runs-on: ['macos-13']\n needs: prep\n\n if: needs.prep.outputs.if_macos_x86_64 == 'true'\n\n\n steps:\n - name: Update Homebrew before installing Rust toolchain\n run: |\n # Homebrew renamed `rustup-init` to `rustup`:\n # https://github.com/Homebrew/homebrew-core/pull/177840\n # But the GitHub Action runner is not updated with this change yet.\n # This caused the later `brew update` in step `Build` to relink Rust\n # toolchain executables, overwriting the custom toolchain installed by\n # `dsherret/rust-toolchain-file`. So let's just run `brew update` early.\n brew update\n\n - uses: actions/checkout@v4\n if: true\n with:\n sparse-checkout: |\n rust-toolchain.toml\n sparse-checkout-cone-mode: false\n\n - name: Install Rust toolchain\n uses: dsherret/rust-toolchain-file@v1\n if: true\n\n - uses: actions/checkout@v4\n with:\n repository: edgedb/edgedb-pkg\n ref: master\n path: edgedb-pkg\n\n - name: Set up Python\n uses: actions/setup-python@v5\n if: true\n with:\n python-version: \"3.12\"\n\n - name: Set up NodeJS\n uses: actions/setup-node@v4\n if: true\n with:\n node-version: '20'\n\n - name: Install dependencies\n if: true\n run: |\n env HOMEBREW_NO_AUTO_UPDATE=1 brew install libmagic\n\n - name: Install an alias\n # This is probably not strictly needed, but sentencepiece build script reports\n # errors without it.\n if: true\n run: |\n printf '#!/bin/sh\\n\\nexec sysctl -n hw.logicalcpu' > /usr/local/bin/nproc\n chmod +x /usr/local/bin/nproc\n\n - name: Build\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"macos\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PKG_PLATFORM_ARCH: \"x86_64\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n BUILD_GENERIC: true\n CMAKE_POLICY_VERSION_MINIMUM: '3.5'\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n run: |\n edgedb-pkg/integration/macos/build.sh\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-macos-x86_64\n path: artifacts/macos-x86_64\n\n build-macos-aarch64:\n runs-on: ['macos-14']\n needs: prep\n\n if: needs.prep.outputs.if_macos_aarch64 == 'true'\n\n\n steps:\n - name: Update Homebrew before installing Rust toolchain\n run: |\n # Homebrew renamed `rustup-init` to `rustup`:\n # https://github.com/Homebrew/homebrew-core/pull/177840\n # But the GitHub Action runner is not updated with this change yet.\n # This caused the later `brew update` in step `Build` to relink Rust\n # toolchain executables, overwriting the custom toolchain installed by\n # `dsherret/rust-toolchain-file`. So let's just run `brew update` early.\n brew update\n\n - uses: actions/checkout@v4\n if: true\n with:\n sparse-checkout: |\n rust-toolchain.toml\n sparse-checkout-cone-mode: false\n\n - name: Install Rust toolchain\n uses: dsherret/rust-toolchain-file@v1\n if: true\n\n - uses: actions/checkout@v4\n with:\n repository: edgedb/edgedb-pkg\n ref: master\n path: edgedb-pkg\n\n - name: Set up Python\n uses: actions/setup-python@v5\n if: true\n with:\n python-version: \"3.12\"\n\n - name: Set up NodeJS\n uses: actions/setup-node@v4\n if: true\n with:\n node-version: '20'\n\n - name: Install dependencies\n if: true\n run: |\n env HOMEBREW_NO_AUTO_UPDATE=1 brew install libmagic\n\n - name: Install an alias\n # This is probably not strictly needed, but sentencepiece build script reports\n # errors without it.\n if: true\n run: |\n printf '#!/bin/sh\\n\\nexec sysctl -n hw.logicalcpu' > /usr/local/bin/nproc\n chmod +x /usr/local/bin/nproc\n\n - name: Build\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"macos\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PKG_PLATFORM_ARCH: \"aarch64\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n BUILD_GENERIC: true\n CMAKE_POLICY_VERSION_MINIMUM: '3.5'\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n run: |\n edgedb-pkg/integration/macos/build.sh\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-macos-aarch64\n path: artifacts/macos-aarch64\n\n test-debian-buster-x86_64:\n needs: [build-debian-buster-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-buster-x86_64\n path: artifacts/debian-buster\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-debian-buster:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"buster\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-debian-buster-aarch64:\n needs: [build-debian-buster-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-buster-aarch64\n path: artifacts/debian-buster\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-debian-buster:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"buster\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-debian-bullseye-x86_64:\n needs: [build-debian-bullseye-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bullseye-x86_64\n path: artifacts/debian-bullseye\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-debian-bullseye:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bullseye\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-debian-bullseye-aarch64:\n needs: [build-debian-bullseye-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bullseye-aarch64\n path: artifacts/debian-bullseye\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-debian-bullseye:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bullseye\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-debian-bookworm-x86_64:\n needs: [build-debian-bookworm-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bookworm-x86_64\n path: artifacts/debian-bookworm\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-debian-bookworm:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bookworm\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-debian-bookworm-aarch64:\n needs: [build-debian-bookworm-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bookworm-aarch64\n path: artifacts/debian-bookworm\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-debian-bookworm:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bookworm\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-ubuntu-focal-x86_64:\n needs: [build-ubuntu-focal-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-focal-x86_64\n path: artifacts/ubuntu-focal\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-ubuntu-focal:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"focal\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-ubuntu-focal-aarch64:\n needs: [build-ubuntu-focal-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-focal-aarch64\n path: artifacts/ubuntu-focal\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-ubuntu-focal:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"focal\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-ubuntu-jammy-x86_64:\n needs: [build-ubuntu-jammy-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-jammy-x86_64\n path: artifacts/ubuntu-jammy\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-ubuntu-jammy:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"jammy\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-ubuntu-jammy-aarch64:\n needs: [build-ubuntu-jammy-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-jammy-aarch64\n path: artifacts/ubuntu-jammy\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-ubuntu-jammy:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"jammy\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-ubuntu-noble-x86_64:\n needs: [build-ubuntu-noble-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-noble-x86_64\n path: artifacts/ubuntu-noble\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-ubuntu-noble:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"noble\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-ubuntu-noble-aarch64:\n needs: [build-ubuntu-noble-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-noble-aarch64\n path: artifacts/ubuntu-noble\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-ubuntu-noble:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"noble\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-centos-8-x86_64:\n needs: [build-centos-8-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-centos-8-x86_64\n path: artifacts/centos-8\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-centos-8:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"centos\"\n PKG_PLATFORM_VERSION: \"8\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-centos-8-aarch64:\n needs: [build-centos-8-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-centos-8-aarch64\n path: artifacts/centos-8\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-centos-8:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"centos\"\n PKG_PLATFORM_VERSION: \"8\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-rockylinux-9-x86_64:\n needs: [build-rockylinux-9-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-rockylinux-9-x86_64\n path: artifacts/rockylinux-9\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-rockylinux-9:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"rockylinux\"\n PKG_PLATFORM_VERSION: \"9\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-rockylinux-9-aarch64:\n needs: [build-rockylinux-9-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-rockylinux-9-aarch64\n path: artifacts/rockylinux-9\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-rockylinux-9:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"rockylinux\"\n PKG_PLATFORM_VERSION: \"9\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-linux-x86_64:\n needs: [build-linux-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linux-x86_64\n path: artifacts/linux-x86_64\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-linux-aarch64:\n needs: [build-linux-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linux-aarch64\n path: artifacts/linux-aarch64\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-linux-aarch64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-linuxmusl-x86_64:\n needs: [build-linuxmusl-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linuxmusl-x86_64\n path: artifacts/linuxmusl-x86_64\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-linuxmusl-x86_64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PKG_PLATFORM_LIBC: \"musl\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-linuxmusl-aarch64:\n needs: [build-linuxmusl-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linuxmusl-aarch64\n path: artifacts/linuxmusl-aarch64\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-linuxmusl-aarch64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PKG_PLATFORM_LIBC: \"musl\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-macos-x86_64:\n needs: [build-macos-x86_64]\n runs-on: ['macos-13']\n\n steps:\n - uses: actions/checkout@v4\n with:\n repository: edgedb/edgedb-pkg\n ref: master\n path: edgedb-pkg\n\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-macos-x86_64\n path: artifacts/macos-x86_64\n\n - name: Test\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"macos\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" test_dump*.py test_backend_*.py test_database.py test_server_*.py test_edgeql_ddl.py test_session.py\n\"\n run: |\n # Bump shmmax and shmall to avoid test failures.\n sudo sysctl -w kern.sysv.shmmax=12582912\n sudo sysctl -w kern.sysv.shmall=12582912\n edgedb-pkg/integration/macos/test.sh\n\n test-macos-aarch64:\n needs: [build-macos-aarch64]\n runs-on: ['macos-14']\n\n steps:\n - uses: actions/checkout@v4\n with:\n repository: edgedb/edgedb-pkg\n ref: master\n path: edgedb-pkg\n\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-macos-aarch64\n path: artifacts/macos-aarch64\n\n - name: Test\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"macos\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n run: |\n edgedb-pkg/integration/macos/test.sh\n\n workflow-notifications:\n if: failure() && github.event_name != 'pull_request'\n name: Notify in Slack on failures\n\n needs:\n - prep\n - build-debian-buster-x86_64\n - test-debian-buster-x86_64\n - build-debian-buster-aarch64\n - test-debian-buster-aarch64\n - build-debian-bullseye-x86_64\n - test-debian-bullseye-x86_64\n - build-debian-bullseye-aarch64\n - test-debian-bullseye-aarch64\n - build-debian-bookworm-x86_64\n - test-debian-bookworm-x86_64\n - build-debian-bookworm-aarch64\n - test-debian-bookworm-aarch64\n - build-ubuntu-focal-x86_64\n - test-ubuntu-focal-x86_64\n - build-ubuntu-focal-aarch64\n - test-ubuntu-focal-aarch64\n - build-ubuntu-jammy-x86_64\n - test-ubuntu-jammy-x86_64\n - build-ubuntu-jammy-aarch64\n - test-ubuntu-jammy-aarch64\n - build-ubuntu-noble-x86_64\n - test-ubuntu-noble-x86_64\n - build-ubuntu-noble-aarch64\n - test-ubuntu-noble-aarch64\n - build-centos-8-x86_64\n - test-centos-8-x86_64\n - build-centos-8-aarch64\n - test-centos-8-aarch64\n - build-rockylinux-9-x86_64\n - test-rockylinux-9-x86_64\n - build-rockylinux-9-aarch64\n - test-rockylinux-9-aarch64\n - build-linux-x86_64\n - test-linux-x86_64\n - build-linux-aarch64\n - test-linux-aarch64\n - build-linuxmusl-x86_64\n - test-linuxmusl-x86_64\n - build-linuxmusl-aarch64\n - test-linuxmusl-aarch64\n - build-macos-x86_64\n - test-macos-x86_64\n - build-macos-aarch64\n - test-macos-aarch64\n runs-on: ubuntu-latest\n permissions:\n actions: 'read'\n steps:\n - name: Slack Workflow Notification\n uses: Gamesight/slack-workflow-status@26a36836c887f260477432e4314ec3490a84f309\n with:\n repo_token: ${{secrets.GITHUB_TOKEN}}\n slack_webhook_url: ${{secrets.ACTIONS_SLACK_WEBHOOK_URL}}\n name: 'Workflow notifications'\n icon_emoji: ':hammer:'\n include_jobs: 'on-failure'\n" }, { "path": ".github/workflows/build.ls-nightly.yml", "content": "name: 'ls: Build and Publish Nightly Packages'\n\non:\n schedule:\n - cron: \"0 1 * * *\"\n workflow_dispatch:\n inputs:\n gelpkg_ref:\n description: \"gel-pkg git ref used to build the packages\"\n default: \"master\"\n metapkg_ref:\n description: \"metapkg git ref used to build the packages\"\n default: \"master\"\n push:\n branches:\n - nightly\n\njobs:\n prep:\n runs-on: ubuntu-latest\n\n outputs:\n\n if_linux_x86_64: ${{ steps.scm.outputs.if_linux_x86_64 }}\n\n if_linux_aarch64: ${{ steps.scm.outputs.if_linux_aarch64 }}\n\n if_linuxmusl_x86_64: ${{ steps.scm.outputs.if_linuxmusl_x86_64 }}\n\n if_linuxmusl_aarch64: ${{ steps.scm.outputs.if_linuxmusl_aarch64 }}\n\n if_macos_x86_64: ${{ steps.scm.outputs.if_macos_x86_64 }}\n\n if_macos_aarch64: ${{ steps.scm.outputs.if_macos_aarch64 }}\n\n\n steps:\n - uses: actions/checkout@v4\n\n\n - name: Determine SCM revision\n id: scm\n shell: bash\n run: |\n rev=$(git rev-parse HEAD)\n jq_filter='.packages[] | select(.basename == \"gel-ls\") | select(.architecture == $ARCH) | .version_details.metadata.scm_revision | . as $rev | select(($rev != null) and ($REV | startswith($rev)))'\n\n key=\"linux-x86_64\"\n val=true\n\n\n idx_file=x86_64-unknown-linux-gnu.nightly.json\n url=https://packages.edgedb.com/archive/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"x86_64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"linux-aarch64\"\n val=true\n\n\n idx_file=aarch64-unknown-linux-gnu.nightly.json\n url=https://packages.edgedb.com/archive/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"aarch64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"linuxmusl-x86_64\"\n val=true\n\n\n idx_file=x86_64-unknown-linux-musl.nightly.json\n url=https://packages.edgedb.com/archive/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"x86_64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"linuxmusl-aarch64\"\n val=true\n\n\n idx_file=aarch64-unknown-linux-musl.nightly.json\n url=https://packages.edgedb.com/archive/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"aarch64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"macos-x86_64\"\n val=true\n\n\n idx_file=x86_64-unknown-linux-gnu.nightly.json\n url=https://packages.edgedb.com/archive/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"x86_64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"macos-aarch64\"\n val=true\n\n\n idx_file=aarch64-unknown-linux-gnu.nightly.json\n url=https://packages.edgedb.com/archive/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"aarch64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n\n\n build-linux-x86_64:\n runs-on: ['self-hosted', 'linux', 'x64']\n needs: prep\n\n if: needs.prep.outputs.if_linux_x86_64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-linux-x86_64:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb_ls:EdgeDBLanguageServer\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_GENERIC: true\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-linux-x86_64\n path: artifacts/linux-x86_64\n\n build-linux-aarch64:\n runs-on: ['self-hosted', 'linux', 'arm64']\n needs: prep\n\n if: needs.prep.outputs.if_linux_aarch64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-linux-aarch64:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb_ls:EdgeDBLanguageServer\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_GENERIC: true\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-linux-aarch64\n path: artifacts/linux-aarch64\n\n build-linuxmusl-x86_64:\n runs-on: ['self-hosted', 'linux', 'x64']\n needs: prep\n\n if: needs.prep.outputs.if_linuxmusl_x86_64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-linuxmusl-x86_64:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb_ls:EdgeDBLanguageServer\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_GENERIC: true\n PKG_PLATFORM_LIBC: \"musl\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-linuxmusl-x86_64\n path: artifacts/linuxmusl-x86_64\n\n build-linuxmusl-aarch64:\n runs-on: ['self-hosted', 'linux', 'arm64']\n needs: prep\n\n if: needs.prep.outputs.if_linuxmusl_aarch64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-linuxmusl-aarch64:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb_ls:EdgeDBLanguageServer\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_GENERIC: true\n PKG_PLATFORM_LIBC: \"musl\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-linuxmusl-aarch64\n path: artifacts/linuxmusl-aarch64\n\n build-macos-x86_64:\n runs-on: ['macos-13']\n needs: prep\n\n if: needs.prep.outputs.if_macos_x86_64 == 'true'\n\n\n steps:\n - name: Update Homebrew before installing Rust toolchain\n run: |\n # Homebrew renamed `rustup-init` to `rustup`:\n # https://github.com/Homebrew/homebrew-core/pull/177840\n # But the GitHub Action runner is not updated with this change yet.\n # This caused the later `brew update` in step `Build` to relink Rust\n # toolchain executables, overwriting the custom toolchain installed by\n # `dsherret/rust-toolchain-file`. So let's just run `brew update` early.\n brew update\n\n - uses: actions/checkout@v4\n if: true\n with:\n sparse-checkout: |\n rust-toolchain.toml\n sparse-checkout-cone-mode: false\n\n - name: Install Rust toolchain\n uses: dsherret/rust-toolchain-file@v1\n if: true\n\n - uses: actions/checkout@v4\n with:\n repository: edgedb/edgedb-pkg\n ref: master\n path: edgedb-pkg\n\n - name: Set up Python\n uses: actions/setup-python@v5\n if: true\n with:\n python-version: \"3.12\"\n\n - name: Set up NodeJS\n uses: actions/setup-node@v4\n if: true\n with:\n node-version: '20'\n\n - name: Install dependencies\n if: true\n run: |\n env HOMEBREW_NO_AUTO_UPDATE=1 brew install libmagic\n\n - name: Install an alias\n # This is probably not strictly needed, but sentencepiece build script reports\n # errors without it.\n if: true\n run: |\n printf '#!/bin/sh\\n\\nexec sysctl -n hw.logicalcpu' > /usr/local/bin/nproc\n chmod +x /usr/local/bin/nproc\n\n - name: Build\n env:\n PACKAGE: \"edgedbpkg.edgedb_ls:EdgeDBLanguageServer\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"macos\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PKG_PLATFORM_ARCH: \"x86_64\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n BUILD_GENERIC: true\n CMAKE_POLICY_VERSION_MINIMUM: '3.5'\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n run: |\n edgedb-pkg/integration/macos/build.sh\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-macos-x86_64\n path: artifacts/macos-x86_64\n\n build-macos-aarch64:\n runs-on: ['macos-14']\n needs: prep\n\n if: needs.prep.outputs.if_macos_aarch64 == 'true'\n\n\n steps:\n - name: Update Homebrew before installing Rust toolchain\n run: |\n # Homebrew renamed `rustup-init` to `rustup`:\n # https://github.com/Homebrew/homebrew-core/pull/177840\n # But the GitHub Action runner is not updated with this change yet.\n # This caused the later `brew update` in step `Build` to relink Rust\n # toolchain executables, overwriting the custom toolchain installed by\n # `dsherret/rust-toolchain-file`. So let's just run `brew update` early.\n brew update\n\n - uses: actions/checkout@v4\n if: true\n with:\n sparse-checkout: |\n rust-toolchain.toml\n sparse-checkout-cone-mode: false\n\n - name: Install Rust toolchain\n uses: dsherret/rust-toolchain-file@v1\n if: true\n\n - uses: actions/checkout@v4\n with:\n repository: edgedb/edgedb-pkg\n ref: master\n path: edgedb-pkg\n\n - name: Set up Python\n uses: actions/setup-python@v5\n if: true\n with:\n python-version: \"3.12\"\n\n - name: Set up NodeJS\n uses: actions/setup-node@v4\n if: true\n with:\n node-version: '20'\n\n - name: Install dependencies\n if: true\n run: |\n env HOMEBREW_NO_AUTO_UPDATE=1 brew install libmagic\n\n - name: Install an alias\n # This is probably not strictly needed, but sentencepiece build script reports\n # errors without it.\n if: true\n run: |\n printf '#!/bin/sh\\n\\nexec sysctl -n hw.logicalcpu' > /usr/local/bin/nproc\n chmod +x /usr/local/bin/nproc\n\n - name: Build\n env:\n PACKAGE: \"edgedbpkg.edgedb_ls:EdgeDBLanguageServer\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"macos\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PKG_PLATFORM_ARCH: \"aarch64\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n BUILD_GENERIC: true\n CMAKE_POLICY_VERSION_MINIMUM: '3.5'\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n run: |\n edgedb-pkg/integration/macos/build.sh\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-macos-aarch64\n path: artifacts/macos-aarch64\n\n publish-linux-x86_64:\n needs: [build-linux-x86_64]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linux-x86_64\n path: artifacts/linux-x86_64\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-linux-x86_64:\n needs: [publish-linux-x86_64]\n runs-on: ['self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linux-x86_64\n path: artifacts/linux-x86_64\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: linux-x86_64\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-linux-x86_64:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-linux-aarch64:\n needs: [build-linux-aarch64]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linux-aarch64\n path: artifacts/linux-aarch64\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-linux-aarch64:\n needs: [publish-linux-aarch64]\n runs-on: ['self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linux-aarch64\n path: artifacts/linux-aarch64\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: linux-aarch64\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-linux-aarch64:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-linuxmusl-x86_64:\n needs: [build-linuxmusl-x86_64]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linuxmusl-x86_64\n path: artifacts/linuxmusl-x86_64\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PKG_PLATFORM_LIBC: \"musl\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-linuxmusl-x86_64:\n needs: [publish-linuxmusl-x86_64]\n runs-on: ['self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linuxmusl-x86_64\n path: artifacts/linuxmusl-x86_64\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: linuxmusl-x86_64\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-linuxmusl-x86_64:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-linuxmusl-aarch64:\n needs: [build-linuxmusl-aarch64]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linuxmusl-aarch64\n path: artifacts/linuxmusl-aarch64\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PKG_PLATFORM_LIBC: \"musl\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-linuxmusl-aarch64:\n needs: [publish-linuxmusl-aarch64]\n runs-on: ['self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linuxmusl-aarch64\n path: artifacts/linuxmusl-aarch64\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: linuxmusl-aarch64\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-linuxmusl-aarch64:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-macos-x86_64:\n needs: [build-macos-x86_64]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-macos-x86_64\n path: artifacts/macos-x86_64\n\n - uses: actions/checkout@v4\n with:\n repository: edgedb/edgedb-pkg\n ref: master\n path: edgedb-pkg\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: macos-x86_64\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"macos\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n publish-macos-aarch64:\n needs: [build-macos-aarch64]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-macos-aarch64\n path: artifacts/macos-aarch64\n\n - uses: actions/checkout@v4\n with:\n repository: edgedb/edgedb-pkg\n ref: master\n path: edgedb-pkg\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: macos-aarch64\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"macos\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n workflow-notifications:\n if: failure() && github.event_name != 'pull_request'\n name: Notify in Slack on failures\n\n needs:\n - prep\n - build-linux-x86_64\n - publish-linux-x86_64\n - check-published-linux-x86_64\n - build-linux-aarch64\n - publish-linux-aarch64\n - check-published-linux-aarch64\n - build-linuxmusl-x86_64\n - publish-linuxmusl-x86_64\n - check-published-linuxmusl-x86_64\n - build-linuxmusl-aarch64\n - publish-linuxmusl-aarch64\n - check-published-linuxmusl-aarch64\n - build-macos-x86_64\n - publish-macos-x86_64\n - build-macos-aarch64\n - publish-macos-aarch64\n runs-on: ubuntu-latest\n permissions:\n actions: 'read'\n steps:\n - name: Slack Workflow Notification\n uses: Gamesight/slack-workflow-status@26a36836c887f260477432e4314ec3490a84f309\n with:\n repo_token: ${{secrets.GITHUB_TOKEN}}\n slack_webhook_url: ${{secrets.ACTIONS_SLACK_WEBHOOK_URL}}\n name: 'Workflow notifications'\n icon_emoji: ':hammer:'\n include_jobs: 'on-failure'\n" }, { "path": ".github/workflows/build.nightly.yml", "content": "name: Build Test and Publish Nightly Packages\n\non:\n schedule:\n - cron: \"0 1 * * *\"\n workflow_dispatch:\n inputs:\n gelpkg_ref:\n description: \"gel-pkg git ref used to build the packages\"\n default: \"master\"\n metapkg_ref:\n description: \"metapkg git ref used to build the packages\"\n default: \"master\"\n push:\n branches:\n - nightly\n\njobs:\n prep:\n runs-on: ubuntu-latest\n\n outputs:\n\n if_debian_buster_x86_64: ${{ steps.scm.outputs.if_debian_buster_x86_64 }}\n\n if_debian_buster_aarch64: ${{ steps.scm.outputs.if_debian_buster_aarch64 }}\n\n if_debian_bullseye_x86_64: ${{ steps.scm.outputs.if_debian_bullseye_x86_64 }}\n\n if_debian_bullseye_aarch64: ${{ steps.scm.outputs.if_debian_bullseye_aarch64 }}\n\n if_debian_bookworm_x86_64: ${{ steps.scm.outputs.if_debian_bookworm_x86_64 }}\n\n if_debian_bookworm_aarch64: ${{ steps.scm.outputs.if_debian_bookworm_aarch64 }}\n\n if_ubuntu_focal_x86_64: ${{ steps.scm.outputs.if_ubuntu_focal_x86_64 }}\n\n if_ubuntu_focal_aarch64: ${{ steps.scm.outputs.if_ubuntu_focal_aarch64 }}\n\n if_ubuntu_jammy_x86_64: ${{ steps.scm.outputs.if_ubuntu_jammy_x86_64 }}\n\n if_ubuntu_jammy_aarch64: ${{ steps.scm.outputs.if_ubuntu_jammy_aarch64 }}\n\n if_ubuntu_noble_x86_64: ${{ steps.scm.outputs.if_ubuntu_noble_x86_64 }}\n\n if_ubuntu_noble_aarch64: ${{ steps.scm.outputs.if_ubuntu_noble_aarch64 }}\n\n if_centos_8_x86_64: ${{ steps.scm.outputs.if_centos_8_x86_64 }}\n\n if_centos_8_aarch64: ${{ steps.scm.outputs.if_centos_8_aarch64 }}\n\n if_rockylinux_9_x86_64: ${{ steps.scm.outputs.if_rockylinux_9_x86_64 }}\n\n if_rockylinux_9_aarch64: ${{ steps.scm.outputs.if_rockylinux_9_aarch64 }}\n\n if_linux_x86_64: ${{ steps.scm.outputs.if_linux_x86_64 }}\n\n if_linux_aarch64: ${{ steps.scm.outputs.if_linux_aarch64 }}\n\n if_linuxmusl_x86_64: ${{ steps.scm.outputs.if_linuxmusl_x86_64 }}\n\n if_linuxmusl_aarch64: ${{ steps.scm.outputs.if_linuxmusl_aarch64 }}\n\n if_macos_x86_64: ${{ steps.scm.outputs.if_macos_x86_64 }}\n\n if_macos_aarch64: ${{ steps.scm.outputs.if_macos_aarch64 }}\n\n\n steps:\n - uses: actions/checkout@v4\n\n\n - name: Determine SCM revision\n id: scm\n shell: bash\n run: |\n rev=$(git rev-parse HEAD)\n jq_filter='.packages[] | select(.basename == \"gel-server\") | select(.architecture == $ARCH) | .version_details.metadata.scm_revision | . as $rev | select(($rev != null) and ($REV | startswith($rev)))'\n\n key=\"debian-buster-x86_64\"\n val=true\n\n\n idx_file=buster.nightly.json\n url=https://packages.edgedb.com/apt/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"x86_64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"debian-buster-aarch64\"\n val=true\n\n\n idx_file=buster.nightly.json\n url=https://packages.edgedb.com/apt/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"aarch64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"debian-bullseye-x86_64\"\n val=true\n\n\n idx_file=bullseye.nightly.json\n url=https://packages.edgedb.com/apt/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"x86_64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"debian-bullseye-aarch64\"\n val=true\n\n\n idx_file=bullseye.nightly.json\n url=https://packages.edgedb.com/apt/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"aarch64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"debian-bookworm-x86_64\"\n val=true\n\n\n idx_file=bookworm.nightly.json\n url=https://packages.edgedb.com/apt/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"x86_64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"debian-bookworm-aarch64\"\n val=true\n\n\n idx_file=bookworm.nightly.json\n url=https://packages.edgedb.com/apt/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"aarch64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"ubuntu-focal-x86_64\"\n val=true\n\n\n idx_file=focal.nightly.json\n url=https://packages.edgedb.com/apt/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"x86_64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"ubuntu-focal-aarch64\"\n val=true\n\n\n idx_file=focal.nightly.json\n url=https://packages.edgedb.com/apt/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"aarch64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"ubuntu-jammy-x86_64\"\n val=true\n\n\n idx_file=jammy.nightly.json\n url=https://packages.edgedb.com/apt/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"x86_64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"ubuntu-jammy-aarch64\"\n val=true\n\n\n idx_file=jammy.nightly.json\n url=https://packages.edgedb.com/apt/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"aarch64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"ubuntu-noble-x86_64\"\n val=true\n\n\n idx_file=noble.nightly.json\n url=https://packages.edgedb.com/apt/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"x86_64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"ubuntu-noble-aarch64\"\n val=true\n\n\n idx_file=noble.nightly.json\n url=https://packages.edgedb.com/apt/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"aarch64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"centos-8-x86_64\"\n val=true\n\n\n idx_file=el8.nightly.json\n url=https://packages.edgedb.com/rpm/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"x86_64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"centos-8-aarch64\"\n val=true\n\n\n idx_file=el8.nightly.json\n url=https://packages.edgedb.com/rpm/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"aarch64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"rockylinux-9-x86_64\"\n val=true\n\n\n idx_file=el9.nightly.json\n url=https://packages.edgedb.com/rpm/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"x86_64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"rockylinux-9-aarch64\"\n val=true\n\n\n idx_file=el9.nightly.json\n url=https://packages.edgedb.com/rpm/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"aarch64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"linux-x86_64\"\n val=true\n\n\n idx_file=x86_64-unknown-linux-gnu.nightly.json\n url=https://packages.edgedb.com/archive/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"x86_64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"linux-aarch64\"\n val=true\n\n\n idx_file=aarch64-unknown-linux-gnu.nightly.json\n url=https://packages.edgedb.com/archive/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"aarch64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"linuxmusl-x86_64\"\n val=true\n\n\n idx_file=x86_64-unknown-linux-musl.nightly.json\n url=https://packages.edgedb.com/archive/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"x86_64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"linuxmusl-aarch64\"\n val=true\n\n\n idx_file=aarch64-unknown-linux-musl.nightly.json\n url=https://packages.edgedb.com/archive/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"aarch64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"macos-x86_64\"\n val=true\n\n\n idx_file=x86_64-unknown-linux-gnu.nightly.json\n url=https://packages.edgedb.com/archive/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"x86_64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n key=\"macos-aarch64\"\n val=true\n\n\n idx_file=aarch64-unknown-linux-gnu.nightly.json\n url=https://packages.edgedb.com/archive/.jsonindexes/$idx_file\n\n\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"aarch64\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n\n\n\n build-debian-buster-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n if: needs.prep.outputs.if_debian_buster_x86_64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-debian-buster:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"buster\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-debian-buster-x86_64\n path: artifacts/debian-buster\n\n build-debian-buster-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n if: needs.prep.outputs.if_debian_buster_aarch64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-debian-buster:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"buster\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-debian-buster-aarch64\n path: artifacts/debian-buster\n\n build-debian-bullseye-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n if: needs.prep.outputs.if_debian_bullseye_x86_64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-debian-bullseye:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bullseye\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-debian-bullseye-x86_64\n path: artifacts/debian-bullseye\n\n build-debian-bullseye-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n if: needs.prep.outputs.if_debian_bullseye_aarch64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-debian-bullseye:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bullseye\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-debian-bullseye-aarch64\n path: artifacts/debian-bullseye\n\n build-debian-bookworm-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n if: needs.prep.outputs.if_debian_bookworm_x86_64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-debian-bookworm:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bookworm\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-debian-bookworm-x86_64\n path: artifacts/debian-bookworm\n\n build-debian-bookworm-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n if: needs.prep.outputs.if_debian_bookworm_aarch64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-debian-bookworm:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bookworm\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-debian-bookworm-aarch64\n path: artifacts/debian-bookworm\n\n build-ubuntu-focal-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n if: needs.prep.outputs.if_ubuntu_focal_x86_64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-ubuntu-focal:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"focal\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-ubuntu-focal-x86_64\n path: artifacts/ubuntu-focal\n\n build-ubuntu-focal-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n if: needs.prep.outputs.if_ubuntu_focal_aarch64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-ubuntu-focal:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"focal\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-ubuntu-focal-aarch64\n path: artifacts/ubuntu-focal\n\n build-ubuntu-jammy-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n if: needs.prep.outputs.if_ubuntu_jammy_x86_64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-ubuntu-jammy:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"jammy\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-ubuntu-jammy-x86_64\n path: artifacts/ubuntu-jammy\n\n build-ubuntu-jammy-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n if: needs.prep.outputs.if_ubuntu_jammy_aarch64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-ubuntu-jammy:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"jammy\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-ubuntu-jammy-aarch64\n path: artifacts/ubuntu-jammy\n\n build-ubuntu-noble-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n if: needs.prep.outputs.if_ubuntu_noble_x86_64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-ubuntu-noble:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"noble\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-ubuntu-noble-x86_64\n path: artifacts/ubuntu-noble\n\n build-ubuntu-noble-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n if: needs.prep.outputs.if_ubuntu_noble_aarch64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-ubuntu-noble:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"noble\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-ubuntu-noble-aarch64\n path: artifacts/ubuntu-noble\n\n build-centos-8-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n if: needs.prep.outputs.if_centos_8_x86_64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-centos-8:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"centos\"\n PKG_PLATFORM_VERSION: \"8\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-centos-8-x86_64\n path: artifacts/centos-8\n\n build-centos-8-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n if: needs.prep.outputs.if_centos_8_aarch64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-centos-8:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"centos\"\n PKG_PLATFORM_VERSION: \"8\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-centos-8-aarch64\n path: artifacts/centos-8\n\n build-rockylinux-9-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n if: needs.prep.outputs.if_rockylinux_9_x86_64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-rockylinux-9:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"rockylinux\"\n PKG_PLATFORM_VERSION: \"9\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-rockylinux-9-x86_64\n path: artifacts/rockylinux-9\n\n build-rockylinux-9-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n if: needs.prep.outputs.if_rockylinux_9_aarch64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-rockylinux-9:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"rockylinux\"\n PKG_PLATFORM_VERSION: \"9\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-rockylinux-9-aarch64\n path: artifacts/rockylinux-9\n\n build-linux-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n if: needs.prep.outputs.if_linux_x86_64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-linux-x86_64:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_GENERIC: true\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-linux-x86_64\n path: artifacts/linux-x86_64\n\n build-linux-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n if: needs.prep.outputs.if_linux_aarch64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-linux-aarch64:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_GENERIC: true\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-linux-aarch64\n path: artifacts/linux-aarch64\n\n build-linuxmusl-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n if: needs.prep.outputs.if_linuxmusl_x86_64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-linuxmusl-x86_64:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_GENERIC: true\n PKG_PLATFORM_LIBC: \"musl\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-linuxmusl-x86_64\n path: artifacts/linuxmusl-x86_64\n\n build-linuxmusl-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n if: needs.prep.outputs.if_linuxmusl_aarch64 == 'true'\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-linuxmusl-aarch64:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_GENERIC: true\n PKG_PLATFORM_LIBC: \"musl\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-linuxmusl-aarch64\n path: artifacts/linuxmusl-aarch64\n\n build-macos-x86_64:\n runs-on: ['macos-13']\n needs: prep\n\n if: needs.prep.outputs.if_macos_x86_64 == 'true'\n\n\n steps:\n - name: Update Homebrew before installing Rust toolchain\n run: |\n # Homebrew renamed `rustup-init` to `rustup`:\n # https://github.com/Homebrew/homebrew-core/pull/177840\n # But the GitHub Action runner is not updated with this change yet.\n # This caused the later `brew update` in step `Build` to relink Rust\n # toolchain executables, overwriting the custom toolchain installed by\n # `dsherret/rust-toolchain-file`. So let's just run `brew update` early.\n brew update\n\n - uses: actions/checkout@v4\n if: true\n with:\n sparse-checkout: |\n rust-toolchain.toml\n sparse-checkout-cone-mode: false\n\n - name: Install Rust toolchain\n uses: dsherret/rust-toolchain-file@v1\n if: true\n\n - uses: actions/checkout@v4\n with:\n repository: edgedb/edgedb-pkg\n ref: master\n path: edgedb-pkg\n\n - name: Set up Python\n uses: actions/setup-python@v5\n if: true\n with:\n python-version: \"3.12\"\n\n - name: Set up NodeJS\n uses: actions/setup-node@v4\n if: true\n with:\n node-version: '20'\n\n - name: Install dependencies\n if: true\n run: |\n env HOMEBREW_NO_AUTO_UPDATE=1 brew install libmagic\n\n - name: Install an alias\n # This is probably not strictly needed, but sentencepiece build script reports\n # errors without it.\n if: true\n run: |\n printf '#!/bin/sh\\n\\nexec sysctl -n hw.logicalcpu' > /usr/local/bin/nproc\n chmod +x /usr/local/bin/nproc\n\n - name: Build\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"macos\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PKG_PLATFORM_ARCH: \"x86_64\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n BUILD_GENERIC: true\n CMAKE_POLICY_VERSION_MINIMUM: '3.5'\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n run: |\n edgedb-pkg/integration/macos/build.sh\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-macos-x86_64\n path: artifacts/macos-x86_64\n\n build-macos-aarch64:\n runs-on: ['macos-14']\n needs: prep\n\n if: needs.prep.outputs.if_macos_aarch64 == 'true'\n\n\n steps:\n - name: Update Homebrew before installing Rust toolchain\n run: |\n # Homebrew renamed `rustup-init` to `rustup`:\n # https://github.com/Homebrew/homebrew-core/pull/177840\n # But the GitHub Action runner is not updated with this change yet.\n # This caused the later `brew update` in step `Build` to relink Rust\n # toolchain executables, overwriting the custom toolchain installed by\n # `dsherret/rust-toolchain-file`. So let's just run `brew update` early.\n brew update\n\n - uses: actions/checkout@v4\n if: true\n with:\n sparse-checkout: |\n rust-toolchain.toml\n sparse-checkout-cone-mode: false\n\n - name: Install Rust toolchain\n uses: dsherret/rust-toolchain-file@v1\n if: true\n\n - uses: actions/checkout@v4\n with:\n repository: edgedb/edgedb-pkg\n ref: master\n path: edgedb-pkg\n\n - name: Set up Python\n uses: actions/setup-python@v5\n if: true\n with:\n python-version: \"3.12\"\n\n - name: Set up NodeJS\n uses: actions/setup-node@v4\n if: true\n with:\n node-version: '20'\n\n - name: Install dependencies\n if: true\n run: |\n env HOMEBREW_NO_AUTO_UPDATE=1 brew install libmagic\n\n - name: Install an alias\n # This is probably not strictly needed, but sentencepiece build script reports\n # errors without it.\n if: true\n run: |\n printf '#!/bin/sh\\n\\nexec sysctl -n hw.logicalcpu' > /usr/local/bin/nproc\n chmod +x /usr/local/bin/nproc\n\n - name: Build\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"macos\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PKG_PLATFORM_ARCH: \"aarch64\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n BUILD_GENERIC: true\n CMAKE_POLICY_VERSION_MINIMUM: '3.5'\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n run: |\n edgedb-pkg/integration/macos/build.sh\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-macos-aarch64\n path: artifacts/macos-aarch64\n\n test-debian-buster-x86_64:\n needs: [build-debian-buster-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-buster-x86_64\n path: artifacts/debian-buster\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-debian-buster:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"buster\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-debian-buster-aarch64:\n needs: [build-debian-buster-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-buster-aarch64\n path: artifacts/debian-buster\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-debian-buster:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"buster\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-debian-bullseye-x86_64:\n needs: [build-debian-bullseye-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bullseye-x86_64\n path: artifacts/debian-bullseye\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-debian-bullseye:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bullseye\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-debian-bullseye-aarch64:\n needs: [build-debian-bullseye-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bullseye-aarch64\n path: artifacts/debian-bullseye\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-debian-bullseye:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bullseye\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-debian-bookworm-x86_64:\n needs: [build-debian-bookworm-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bookworm-x86_64\n path: artifacts/debian-bookworm\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-debian-bookworm:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bookworm\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-debian-bookworm-aarch64:\n needs: [build-debian-bookworm-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bookworm-aarch64\n path: artifacts/debian-bookworm\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-debian-bookworm:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bookworm\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-ubuntu-focal-x86_64:\n needs: [build-ubuntu-focal-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-focal-x86_64\n path: artifacts/ubuntu-focal\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-ubuntu-focal:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"focal\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-ubuntu-focal-aarch64:\n needs: [build-ubuntu-focal-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-focal-aarch64\n path: artifacts/ubuntu-focal\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-ubuntu-focal:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"focal\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-ubuntu-jammy-x86_64:\n needs: [build-ubuntu-jammy-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-jammy-x86_64\n path: artifacts/ubuntu-jammy\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-ubuntu-jammy:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"jammy\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-ubuntu-jammy-aarch64:\n needs: [build-ubuntu-jammy-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-jammy-aarch64\n path: artifacts/ubuntu-jammy\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-ubuntu-jammy:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"jammy\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-ubuntu-noble-x86_64:\n needs: [build-ubuntu-noble-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-noble-x86_64\n path: artifacts/ubuntu-noble\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-ubuntu-noble:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"noble\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-ubuntu-noble-aarch64:\n needs: [build-ubuntu-noble-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-noble-aarch64\n path: artifacts/ubuntu-noble\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-ubuntu-noble:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"noble\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-centos-8-x86_64:\n needs: [build-centos-8-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-centos-8-x86_64\n path: artifacts/centos-8\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-centos-8:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"centos\"\n PKG_PLATFORM_VERSION: \"8\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-centos-8-aarch64:\n needs: [build-centos-8-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-centos-8-aarch64\n path: artifacts/centos-8\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-centos-8:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"centos\"\n PKG_PLATFORM_VERSION: \"8\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-rockylinux-9-x86_64:\n needs: [build-rockylinux-9-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-rockylinux-9-x86_64\n path: artifacts/rockylinux-9\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-rockylinux-9:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"rockylinux\"\n PKG_PLATFORM_VERSION: \"9\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-rockylinux-9-aarch64:\n needs: [build-rockylinux-9-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-rockylinux-9-aarch64\n path: artifacts/rockylinux-9\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-rockylinux-9:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"rockylinux\"\n PKG_PLATFORM_VERSION: \"9\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-linux-x86_64:\n needs: [build-linux-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linux-x86_64\n path: artifacts/linux-x86_64\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-linux-aarch64:\n needs: [build-linux-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linux-aarch64\n path: artifacts/linux-aarch64\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-linux-aarch64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-linuxmusl-x86_64:\n needs: [build-linuxmusl-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linuxmusl-x86_64\n path: artifacts/linuxmusl-x86_64\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-linuxmusl-x86_64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PKG_PLATFORM_LIBC: \"musl\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-linuxmusl-aarch64:\n needs: [build-linuxmusl-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linuxmusl-aarch64\n path: artifacts/linuxmusl-aarch64\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-linuxmusl-aarch64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PKG_PLATFORM_LIBC: \"musl\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-macos-x86_64:\n needs: [build-macos-x86_64]\n runs-on: ['macos-13']\n\n steps:\n - uses: actions/checkout@v4\n with:\n repository: edgedb/edgedb-pkg\n ref: master\n path: edgedb-pkg\n\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-macos-x86_64\n path: artifacts/macos-x86_64\n\n - name: Test\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"macos\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" test_dump*.py test_backend_*.py test_database.py test_server_*.py test_edgeql_ddl.py test_session.py\n\"\n run: |\n # Bump shmmax and shmall to avoid test failures.\n sudo sysctl -w kern.sysv.shmmax=12582912\n sudo sysctl -w kern.sysv.shmall=12582912\n edgedb-pkg/integration/macos/test.sh\n\n test-macos-aarch64:\n needs: [build-macos-aarch64]\n runs-on: ['macos-14']\n\n steps:\n - uses: actions/checkout@v4\n with:\n repository: edgedb/edgedb-pkg\n ref: master\n path: edgedb-pkg\n\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-macos-aarch64\n path: artifacts/macos-aarch64\n\n - name: Test\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"macos\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n run: |\n edgedb-pkg/integration/macos/test.sh\n\n publish-debian-buster-x86_64:\n needs: [test-debian-buster-x86_64]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-buster-x86_64\n path: artifacts/debian-buster\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"buster\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-debian-buster-x86_64:\n needs: [publish-debian-buster-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-buster-x86_64\n path: artifacts/debian-buster\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: debian-buster\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-debian-buster:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"buster\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-debian-buster-aarch64:\n needs: [test-debian-buster-aarch64]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-buster-aarch64\n path: artifacts/debian-buster\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"buster\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-debian-buster-aarch64:\n needs: [publish-debian-buster-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-buster-aarch64\n path: artifacts/debian-buster\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: debian-buster\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-debian-buster:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"buster\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-debian-bullseye-x86_64:\n needs: [test-debian-bullseye-x86_64]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bullseye-x86_64\n path: artifacts/debian-bullseye\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bullseye\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-debian-bullseye-x86_64:\n needs: [publish-debian-bullseye-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bullseye-x86_64\n path: artifacts/debian-bullseye\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: debian-bullseye\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-debian-bullseye:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bullseye\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-debian-bullseye-aarch64:\n needs: [test-debian-bullseye-aarch64]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bullseye-aarch64\n path: artifacts/debian-bullseye\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bullseye\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-debian-bullseye-aarch64:\n needs: [publish-debian-bullseye-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bullseye-aarch64\n path: artifacts/debian-bullseye\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: debian-bullseye\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-debian-bullseye:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bullseye\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-debian-bookworm-x86_64:\n needs: [test-debian-bookworm-x86_64]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bookworm-x86_64\n path: artifacts/debian-bookworm\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bookworm\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-debian-bookworm-x86_64:\n needs: [publish-debian-bookworm-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bookworm-x86_64\n path: artifacts/debian-bookworm\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: debian-bookworm\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-debian-bookworm:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bookworm\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-debian-bookworm-aarch64:\n needs: [test-debian-bookworm-aarch64]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bookworm-aarch64\n path: artifacts/debian-bookworm\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bookworm\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-debian-bookworm-aarch64:\n needs: [publish-debian-bookworm-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bookworm-aarch64\n path: artifacts/debian-bookworm\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: debian-bookworm\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-debian-bookworm:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bookworm\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-ubuntu-focal-x86_64:\n needs: [test-ubuntu-focal-x86_64]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-focal-x86_64\n path: artifacts/ubuntu-focal\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"focal\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-ubuntu-focal-x86_64:\n needs: [publish-ubuntu-focal-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-focal-x86_64\n path: artifacts/ubuntu-focal\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: ubuntu-focal\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-ubuntu-focal:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"focal\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-ubuntu-focal-aarch64:\n needs: [test-ubuntu-focal-aarch64]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-focal-aarch64\n path: artifacts/ubuntu-focal\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"focal\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-ubuntu-focal-aarch64:\n needs: [publish-ubuntu-focal-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-focal-aarch64\n path: artifacts/ubuntu-focal\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: ubuntu-focal\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-ubuntu-focal:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"focal\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-ubuntu-jammy-x86_64:\n needs: [test-ubuntu-jammy-x86_64]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-jammy-x86_64\n path: artifacts/ubuntu-jammy\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"jammy\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-ubuntu-jammy-x86_64:\n needs: [publish-ubuntu-jammy-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-jammy-x86_64\n path: artifacts/ubuntu-jammy\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: ubuntu-jammy\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-ubuntu-jammy:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"jammy\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-ubuntu-jammy-aarch64:\n needs: [test-ubuntu-jammy-aarch64]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-jammy-aarch64\n path: artifacts/ubuntu-jammy\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"jammy\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-ubuntu-jammy-aarch64:\n needs: [publish-ubuntu-jammy-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-jammy-aarch64\n path: artifacts/ubuntu-jammy\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: ubuntu-jammy\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-ubuntu-jammy:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"jammy\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-ubuntu-noble-x86_64:\n needs: [test-ubuntu-noble-x86_64]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-noble-x86_64\n path: artifacts/ubuntu-noble\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"noble\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-ubuntu-noble-x86_64:\n needs: [publish-ubuntu-noble-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-noble-x86_64\n path: artifacts/ubuntu-noble\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: ubuntu-noble\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-ubuntu-noble:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"noble\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-ubuntu-noble-aarch64:\n needs: [test-ubuntu-noble-aarch64]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-noble-aarch64\n path: artifacts/ubuntu-noble\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"noble\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-ubuntu-noble-aarch64:\n needs: [publish-ubuntu-noble-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-noble-aarch64\n path: artifacts/ubuntu-noble\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: ubuntu-noble\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-ubuntu-noble:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"noble\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-centos-8-x86_64:\n needs: [test-centos-8-x86_64]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-centos-8-x86_64\n path: artifacts/centos-8\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"centos\"\n PKG_PLATFORM_VERSION: \"8\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-centos-8-x86_64:\n needs: [publish-centos-8-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-centos-8-x86_64\n path: artifacts/centos-8\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: centos-8\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-centos-8:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"centos\"\n PKG_PLATFORM_VERSION: \"8\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-centos-8-aarch64:\n needs: [test-centos-8-aarch64]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-centos-8-aarch64\n path: artifacts/centos-8\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"centos\"\n PKG_PLATFORM_VERSION: \"8\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-centos-8-aarch64:\n needs: [publish-centos-8-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-centos-8-aarch64\n path: artifacts/centos-8\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: centos-8\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-centos-8:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"centos\"\n PKG_PLATFORM_VERSION: \"8\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-rockylinux-9-x86_64:\n needs: [test-rockylinux-9-x86_64]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-rockylinux-9-x86_64\n path: artifacts/rockylinux-9\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"rockylinux\"\n PKG_PLATFORM_VERSION: \"9\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-rockylinux-9-x86_64:\n needs: [publish-rockylinux-9-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-rockylinux-9-x86_64\n path: artifacts/rockylinux-9\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: rockylinux-9\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-rockylinux-9:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"rockylinux\"\n PKG_PLATFORM_VERSION: \"9\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-rockylinux-9-aarch64:\n needs: [test-rockylinux-9-aarch64]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-rockylinux-9-aarch64\n path: artifacts/rockylinux-9\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"rockylinux\"\n PKG_PLATFORM_VERSION: \"9\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-rockylinux-9-aarch64:\n needs: [publish-rockylinux-9-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-rockylinux-9-aarch64\n path: artifacts/rockylinux-9\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: rockylinux-9\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-rockylinux-9:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"rockylinux\"\n PKG_PLATFORM_VERSION: \"9\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-linux-x86_64:\n needs: [test-linux-x86_64]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linux-x86_64\n path: artifacts/linux-x86_64\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-linux-x86_64:\n needs: [publish-linux-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linux-x86_64\n path: artifacts/linux-x86_64\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: linux-x86_64\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-linux-x86_64:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-linux-aarch64:\n needs: [test-linux-aarch64]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linux-aarch64\n path: artifacts/linux-aarch64\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-linux-aarch64:\n needs: [publish-linux-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linux-aarch64\n path: artifacts/linux-aarch64\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: linux-aarch64\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-linux-aarch64:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-linuxmusl-x86_64:\n needs: [test-linuxmusl-x86_64]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linuxmusl-x86_64\n path: artifacts/linuxmusl-x86_64\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PKG_PLATFORM_LIBC: \"musl\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-linuxmusl-x86_64:\n needs: [publish-linuxmusl-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linuxmusl-x86_64\n path: artifacts/linuxmusl-x86_64\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: linuxmusl-x86_64\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-linuxmusl-x86_64:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-linuxmusl-aarch64:\n needs: [test-linuxmusl-aarch64]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linuxmusl-aarch64\n path: artifacts/linuxmusl-aarch64\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PKG_PLATFORM_LIBC: \"musl\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-linuxmusl-aarch64:\n needs: [publish-linuxmusl-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linuxmusl-aarch64\n path: artifacts/linuxmusl-aarch64\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: linuxmusl-aarch64\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-linuxmusl-aarch64:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"nightly\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-macos-x86_64:\n needs: [test-macos-x86_64]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-macos-x86_64\n path: artifacts/macos-x86_64\n\n - uses: actions/checkout@v4\n with:\n repository: edgedb/edgedb-pkg\n ref: master\n path: edgedb-pkg\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: macos-x86_64\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"macos\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n publish-macos-aarch64:\n needs: [test-macos-aarch64]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-macos-aarch64\n path: artifacts/macos-aarch64\n\n - uses: actions/checkout@v4\n with:\n repository: edgedb/edgedb-pkg\n ref: master\n path: edgedb-pkg\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: macos-aarch64\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"nightly\"\n PKG_PLATFORM: \"macos\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n publish-docker:\n needs:\n - check-published-debian-bookworm-x86_64\n - check-published-debian-bookworm-aarch64\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/checkout@v4\n with:\n repository: geldata/gel-docker\n ref: master\n path: dockerfile\n\n - name: Login to Docker Hub\n uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0\n with:\n username: ${{ secrets.DOCKER_USERNAME }}\n password: ${{ secrets.DOCKER_PASSWORD }}\n\n - name: Login to GitHub Container Registry\n uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0\n with:\n registry: ghcr.io\n username: \"edgedb-ci\"\n password: ${{ secrets.GITHUB_CI_BOT_TOKEN }}\n\n - env:\n VERSION_SLOT: \"${{ needs.check-published-debian-bookworm-x86_64.outputs.version-slot }}\"\n VERSION_CORE: \"${{ needs.check-published-debian-bookworm-x86_64.outputs.version-core }}\"\n CATALOG_VERSION: \"${{ needs.check-published-debian-bookworm-x86_64.outputs.catalog-version }}\"\n PKG_SUBDIST: \"nightly\"\n id: tags\n run: |\n set -e\n\n url='https://registry.hub.docker.com/v2/repositories/geldata/gel/tags?page_size=100'\n repo_tags=$(\n while [ -n \"$url\" ]; do\n resp=$(curl -L -s \"$url\")\n url=$(echo \"$resp\" | jq -r \".next\")\n if [ \"$url\" = \"null\" ] || [ -z \"$url\" ]; then\n break\n fi\n echo \"$resp\" | jq -r '.\"results\"[][\"name\"]'\n done | grep \"^[[:digit:]]\\+.*\" | grep -v \"alpha\\|beta\\|rc\" || :\n )\n\n tags=()\n\n if [ \"$PKG_SUBDIST\" = \"nightly\" ]; then\n tags+=(\n \"nightly\"\n \"nightly_${VERSION_SLOT}_cv${CATALOG_VERSION}\"\n )\n else\n tags+=( \"$VERSION_CORE\" )\n\n top=$(printf \"%s\\n%s\\n\" \"$VERSION_CORE\" \"$repo_tags\" \\\n | grep \"^${VERSION_SLOT}[\\.-]\" \\\n | sort --version-sort --reverse | head -n 1)\n if [ \"$top\" == \"$VERSION_CORE\" ]; then\n tags+=( \"$VERSION_SLOT\" )\n fi\n\n if [ -z \"$PKG_SUBDIST\" ]; then\n top=$(printf \"%s\\n%s\\n\" \"$VERSION_CORE\" \"$repo_tags\" \\\n | sort --version-sort --reverse | head -n 1)\n if [ \"$top\" == \"$VERSION_CORE\" ]; then\n tags+=( \"latest\" )\n fi\n fi\n fi\n\n fq_tags=()\n images=(\"geldata/gel\" \"ghcr.io/geldata/gel\")\n\n for image in \"${images[@]}\"; do\n fq_tags+=(\"${tags[@]/#/${image}:}\")\n done\n\n IFS=,\n echo \"tags=${fq_tags[*]}\" >> $GITHUB_OUTPUT\n\n - name: Set up QEMU\n uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0\n\n - name: Set up Docker Buildx\n uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0\n\n - name: Build and Publish Docker Image\n uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6.10.0\n with:\n push: true\n provenance: mode=max\n tags: \"${{ steps.tags.outputs.tags }}\"\n context: dockerfile\n build-args: |\n version=${{ needs.check-published-debian-bookworm-x86_64.outputs.version-slot }}\n exact_version=${{ needs.check-published-debian-bookworm-x86_64.outputs.version-core }}\n subdist=nightly\n platforms: linux/amd64,linux/arm64\n\n workflow-notifications:\n if: failure() && github.event_name != 'pull_request'\n name: Notify in Slack on failures\n\n needs:\n - prep\n - build-debian-buster-x86_64\n - test-debian-buster-x86_64\n - publish-debian-buster-x86_64\n - check-published-debian-buster-x86_64\n - build-debian-buster-aarch64\n - test-debian-buster-aarch64\n - publish-debian-buster-aarch64\n - check-published-debian-buster-aarch64\n - build-debian-bullseye-x86_64\n - test-debian-bullseye-x86_64\n - publish-debian-bullseye-x86_64\n - check-published-debian-bullseye-x86_64\n - build-debian-bullseye-aarch64\n - test-debian-bullseye-aarch64\n - publish-debian-bullseye-aarch64\n - check-published-debian-bullseye-aarch64\n - build-debian-bookworm-x86_64\n - test-debian-bookworm-x86_64\n - publish-debian-bookworm-x86_64\n - check-published-debian-bookworm-x86_64\n - build-debian-bookworm-aarch64\n - test-debian-bookworm-aarch64\n - publish-debian-bookworm-aarch64\n - check-published-debian-bookworm-aarch64\n - build-ubuntu-focal-x86_64\n - test-ubuntu-focal-x86_64\n - publish-ubuntu-focal-x86_64\n - check-published-ubuntu-focal-x86_64\n - build-ubuntu-focal-aarch64\n - test-ubuntu-focal-aarch64\n - publish-ubuntu-focal-aarch64\n - check-published-ubuntu-focal-aarch64\n - build-ubuntu-jammy-x86_64\n - test-ubuntu-jammy-x86_64\n - publish-ubuntu-jammy-x86_64\n - check-published-ubuntu-jammy-x86_64\n - build-ubuntu-jammy-aarch64\n - test-ubuntu-jammy-aarch64\n - publish-ubuntu-jammy-aarch64\n - check-published-ubuntu-jammy-aarch64\n - build-ubuntu-noble-x86_64\n - test-ubuntu-noble-x86_64\n - publish-ubuntu-noble-x86_64\n - check-published-ubuntu-noble-x86_64\n - build-ubuntu-noble-aarch64\n - test-ubuntu-noble-aarch64\n - publish-ubuntu-noble-aarch64\n - check-published-ubuntu-noble-aarch64\n - build-centos-8-x86_64\n - test-centos-8-x86_64\n - publish-centos-8-x86_64\n - check-published-centos-8-x86_64\n - build-centos-8-aarch64\n - test-centos-8-aarch64\n - publish-centos-8-aarch64\n - check-published-centos-8-aarch64\n - build-rockylinux-9-x86_64\n - test-rockylinux-9-x86_64\n - publish-rockylinux-9-x86_64\n - check-published-rockylinux-9-x86_64\n - build-rockylinux-9-aarch64\n - test-rockylinux-9-aarch64\n - publish-rockylinux-9-aarch64\n - check-published-rockylinux-9-aarch64\n - build-linux-x86_64\n - test-linux-x86_64\n - publish-linux-x86_64\n - check-published-linux-x86_64\n - build-linux-aarch64\n - test-linux-aarch64\n - publish-linux-aarch64\n - check-published-linux-aarch64\n - build-linuxmusl-x86_64\n - test-linuxmusl-x86_64\n - publish-linuxmusl-x86_64\n - check-published-linuxmusl-x86_64\n - build-linuxmusl-aarch64\n - test-linuxmusl-aarch64\n - publish-linuxmusl-aarch64\n - check-published-linuxmusl-aarch64\n - build-macos-x86_64\n - test-macos-x86_64\n - publish-macos-x86_64\n - build-macos-aarch64\n - test-macos-aarch64\n - publish-macos-aarch64\n - publish-docker\n runs-on: ubuntu-latest\n permissions:\n actions: 'read'\n steps:\n - name: Slack Workflow Notification\n uses: Gamesight/slack-workflow-status@26a36836c887f260477432e4314ec3490a84f309\n with:\n repo_token: ${{secrets.GITHUB_TOKEN}}\n slack_webhook_url: ${{secrets.ACTIONS_SLACK_WEBHOOK_URL}}\n name: 'Workflow notifications'\n icon_emoji: ':hammer:'\n include_jobs: 'on-failure'\n" }, { "path": ".github/workflows/build.release.yml", "content": "name: Build Test and Publish a Release\n\non:\n workflow_dispatch:\n inputs:\n gelpkg_ref:\n description: \"gel-pkg git ref used to build the packages\"\n default: \"master\"\n metapkg_ref:\n description: \"metapkg git ref used to build the packages\"\n default: \"master\"\n\njobs:\n prep:\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/checkout@v4\n\n\n\n build-debian-buster-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-debian-buster:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"buster\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-debian-buster-x86_64\n path: artifacts/debian-buster\n\n build-debian-buster-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-debian-buster:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"buster\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-debian-buster-aarch64\n path: artifacts/debian-buster\n\n build-debian-bullseye-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-debian-bullseye:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bullseye\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-debian-bullseye-x86_64\n path: artifacts/debian-bullseye\n\n build-debian-bullseye-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-debian-bullseye:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bullseye\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-debian-bullseye-aarch64\n path: artifacts/debian-bullseye\n\n build-debian-bookworm-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-debian-bookworm:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bookworm\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-debian-bookworm-x86_64\n path: artifacts/debian-bookworm\n\n build-debian-bookworm-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-debian-bookworm:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bookworm\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-debian-bookworm-aarch64\n path: artifacts/debian-bookworm\n\n build-ubuntu-focal-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-ubuntu-focal:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"focal\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-ubuntu-focal-x86_64\n path: artifacts/ubuntu-focal\n\n build-ubuntu-focal-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-ubuntu-focal:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"focal\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-ubuntu-focal-aarch64\n path: artifacts/ubuntu-focal\n\n build-ubuntu-jammy-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-ubuntu-jammy:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"jammy\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-ubuntu-jammy-x86_64\n path: artifacts/ubuntu-jammy\n\n build-ubuntu-jammy-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-ubuntu-jammy:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"jammy\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-ubuntu-jammy-aarch64\n path: artifacts/ubuntu-jammy\n\n build-ubuntu-noble-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-ubuntu-noble:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"noble\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-ubuntu-noble-x86_64\n path: artifacts/ubuntu-noble\n\n build-ubuntu-noble-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-ubuntu-noble:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"noble\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-ubuntu-noble-aarch64\n path: artifacts/ubuntu-noble\n\n build-centos-8-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-centos-8:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_PLATFORM: \"centos\"\n PKG_PLATFORM_VERSION: \"8\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-centos-8-x86_64\n path: artifacts/centos-8\n\n build-centos-8-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-centos-8:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_PLATFORM: \"centos\"\n PKG_PLATFORM_VERSION: \"8\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-centos-8-aarch64\n path: artifacts/centos-8\n\n build-rockylinux-9-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-rockylinux-9:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_PLATFORM: \"rockylinux\"\n PKG_PLATFORM_VERSION: \"9\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-rockylinux-9-x86_64\n path: artifacts/rockylinux-9\n\n build-rockylinux-9-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-rockylinux-9:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_PLATFORM: \"rockylinux\"\n PKG_PLATFORM_VERSION: \"9\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-rockylinux-9-aarch64\n path: artifacts/rockylinux-9\n\n build-linux-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-linux-x86_64:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n BUILD_GENERIC: true\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-linux-x86_64\n path: artifacts/linux-x86_64\n\n build-linux-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-linux-aarch64:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n BUILD_GENERIC: true\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-linux-aarch64\n path: artifacts/linux-aarch64\n\n build-linuxmusl-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-linuxmusl-x86_64:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n BUILD_GENERIC: true\n PKG_PLATFORM_LIBC: \"musl\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-linuxmusl-x86_64\n path: artifacts/linuxmusl-x86_64\n\n build-linuxmusl-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-linuxmusl-aarch64:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n BUILD_GENERIC: true\n PKG_PLATFORM_LIBC: \"musl\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-linuxmusl-aarch64\n path: artifacts/linuxmusl-aarch64\n\n build-macos-x86_64:\n runs-on: ['macos-13']\n needs: prep\n\n\n steps:\n - name: Update Homebrew before installing Rust toolchain\n run: |\n # Homebrew renamed `rustup-init` to `rustup`:\n # https://github.com/Homebrew/homebrew-core/pull/177840\n # But the GitHub Action runner is not updated with this change yet.\n # This caused the later `brew update` in step `Build` to relink Rust\n # toolchain executables, overwriting the custom toolchain installed by\n # `dsherret/rust-toolchain-file`. So let's just run `brew update` early.\n brew update\n\n - uses: actions/checkout@v4\n if: true\n with:\n sparse-checkout: |\n rust-toolchain.toml\n sparse-checkout-cone-mode: false\n\n - name: Install Rust toolchain\n uses: dsherret/rust-toolchain-file@v1\n if: true\n\n - uses: actions/checkout@v4\n with:\n repository: edgedb/edgedb-pkg\n ref: master\n path: edgedb-pkg\n\n - name: Set up Python\n uses: actions/setup-python@v5\n if: true\n with:\n python-version: \"3.12\"\n\n - name: Set up NodeJS\n uses: actions/setup-node@v4\n if: true\n with:\n node-version: '20'\n\n - name: Install dependencies\n if: true\n run: |\n env HOMEBREW_NO_AUTO_UPDATE=1 brew install libmagic\n\n - name: Install an alias\n # This is probably not strictly needed, but sentencepiece build script reports\n # errors without it.\n if: true\n run: |\n printf '#!/bin/sh\\n\\nexec sysctl -n hw.logicalcpu' > /usr/local/bin/nproc\n chmod +x /usr/local/bin/nproc\n\n - name: Build\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n BUILD_IS_RELEASE: \"true\"\n PKG_REVISION: \"\"\n PKG_PLATFORM: \"macos\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PKG_PLATFORM_ARCH: \"x86_64\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n BUILD_GENERIC: true\n CMAKE_POLICY_VERSION_MINIMUM: '3.5'\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n run: |\n edgedb-pkg/integration/macos/build.sh\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-macos-x86_64\n path: artifacts/macos-x86_64\n\n build-macos-aarch64:\n runs-on: ['macos-14']\n needs: prep\n\n\n steps:\n - name: Update Homebrew before installing Rust toolchain\n run: |\n # Homebrew renamed `rustup-init` to `rustup`:\n # https://github.com/Homebrew/homebrew-core/pull/177840\n # But the GitHub Action runner is not updated with this change yet.\n # This caused the later `brew update` in step `Build` to relink Rust\n # toolchain executables, overwriting the custom toolchain installed by\n # `dsherret/rust-toolchain-file`. So let's just run `brew update` early.\n brew update\n\n - uses: actions/checkout@v4\n if: true\n with:\n sparse-checkout: |\n rust-toolchain.toml\n sparse-checkout-cone-mode: false\n\n - name: Install Rust toolchain\n uses: dsherret/rust-toolchain-file@v1\n if: true\n\n - uses: actions/checkout@v4\n with:\n repository: edgedb/edgedb-pkg\n ref: master\n path: edgedb-pkg\n\n - name: Set up Python\n uses: actions/setup-python@v5\n if: true\n with:\n python-version: \"3.12\"\n\n - name: Set up NodeJS\n uses: actions/setup-node@v4\n if: true\n with:\n node-version: '20'\n\n - name: Install dependencies\n if: true\n run: |\n env HOMEBREW_NO_AUTO_UPDATE=1 brew install libmagic\n\n - name: Install an alias\n # This is probably not strictly needed, but sentencepiece build script reports\n # errors without it.\n if: true\n run: |\n printf '#!/bin/sh\\n\\nexec sysctl -n hw.logicalcpu' > /usr/local/bin/nproc\n chmod +x /usr/local/bin/nproc\n\n - name: Build\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n BUILD_IS_RELEASE: \"true\"\n PKG_REVISION: \"\"\n PKG_PLATFORM: \"macos\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PKG_PLATFORM_ARCH: \"aarch64\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n BUILD_GENERIC: true\n CMAKE_POLICY_VERSION_MINIMUM: '3.5'\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n run: |\n edgedb-pkg/integration/macos/build.sh\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-macos-aarch64\n path: artifacts/macos-aarch64\n\n test-debian-buster-x86_64:\n needs: [build-debian-buster-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-buster-x86_64\n path: artifacts/debian-buster\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-debian-buster:latest\n env:\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"buster\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-debian-buster-aarch64:\n needs: [build-debian-buster-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-buster-aarch64\n path: artifacts/debian-buster\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-debian-buster:latest\n env:\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"buster\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-debian-bullseye-x86_64:\n needs: [build-debian-bullseye-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bullseye-x86_64\n path: artifacts/debian-bullseye\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-debian-bullseye:latest\n env:\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bullseye\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-debian-bullseye-aarch64:\n needs: [build-debian-bullseye-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bullseye-aarch64\n path: artifacts/debian-bullseye\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-debian-bullseye:latest\n env:\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bullseye\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-debian-bookworm-x86_64:\n needs: [build-debian-bookworm-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bookworm-x86_64\n path: artifacts/debian-bookworm\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-debian-bookworm:latest\n env:\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bookworm\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-debian-bookworm-aarch64:\n needs: [build-debian-bookworm-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bookworm-aarch64\n path: artifacts/debian-bookworm\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-debian-bookworm:latest\n env:\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bookworm\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-ubuntu-focal-x86_64:\n needs: [build-ubuntu-focal-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-focal-x86_64\n path: artifacts/ubuntu-focal\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-ubuntu-focal:latest\n env:\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"focal\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-ubuntu-focal-aarch64:\n needs: [build-ubuntu-focal-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-focal-aarch64\n path: artifacts/ubuntu-focal\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-ubuntu-focal:latest\n env:\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"focal\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-ubuntu-jammy-x86_64:\n needs: [build-ubuntu-jammy-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-jammy-x86_64\n path: artifacts/ubuntu-jammy\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-ubuntu-jammy:latest\n env:\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"jammy\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-ubuntu-jammy-aarch64:\n needs: [build-ubuntu-jammy-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-jammy-aarch64\n path: artifacts/ubuntu-jammy\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-ubuntu-jammy:latest\n env:\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"jammy\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-ubuntu-noble-x86_64:\n needs: [build-ubuntu-noble-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-noble-x86_64\n path: artifacts/ubuntu-noble\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-ubuntu-noble:latest\n env:\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"noble\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-ubuntu-noble-aarch64:\n needs: [build-ubuntu-noble-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-noble-aarch64\n path: artifacts/ubuntu-noble\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-ubuntu-noble:latest\n env:\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"noble\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-centos-8-x86_64:\n needs: [build-centos-8-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-centos-8-x86_64\n path: artifacts/centos-8\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-centos-8:latest\n env:\n PKG_PLATFORM: \"centos\"\n PKG_PLATFORM_VERSION: \"8\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-centos-8-aarch64:\n needs: [build-centos-8-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-centos-8-aarch64\n path: artifacts/centos-8\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-centos-8:latest\n env:\n PKG_PLATFORM: \"centos\"\n PKG_PLATFORM_VERSION: \"8\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-rockylinux-9-x86_64:\n needs: [build-rockylinux-9-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-rockylinux-9-x86_64\n path: artifacts/rockylinux-9\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-rockylinux-9:latest\n env:\n PKG_PLATFORM: \"rockylinux\"\n PKG_PLATFORM_VERSION: \"9\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-rockylinux-9-aarch64:\n needs: [build-rockylinux-9-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-rockylinux-9-aarch64\n path: artifacts/rockylinux-9\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-rockylinux-9:latest\n env:\n PKG_PLATFORM: \"rockylinux\"\n PKG_PLATFORM_VERSION: \"9\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-linux-x86_64:\n needs: [build-linux-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linux-x86_64\n path: artifacts/linux-x86_64\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-linux-x86_64:latest\n env:\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-linux-aarch64:\n needs: [build-linux-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linux-aarch64\n path: artifacts/linux-aarch64\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-linux-aarch64:latest\n env:\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-linuxmusl-x86_64:\n needs: [build-linuxmusl-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linuxmusl-x86_64\n path: artifacts/linuxmusl-x86_64\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-linuxmusl-x86_64:latest\n env:\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PKG_PLATFORM_LIBC: \"musl\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-linuxmusl-aarch64:\n needs: [build-linuxmusl-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linuxmusl-aarch64\n path: artifacts/linuxmusl-aarch64\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-linuxmusl-aarch64:latest\n env:\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PKG_PLATFORM_LIBC: \"musl\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-macos-x86_64:\n needs: [build-macos-x86_64]\n runs-on: ['macos-13']\n\n steps:\n - uses: actions/checkout@v4\n with:\n repository: edgedb/edgedb-pkg\n ref: master\n path: edgedb-pkg\n\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-macos-x86_64\n path: artifacts/macos-x86_64\n\n - name: Test\n env:\n PKG_PLATFORM: \"macos\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" test_dump*.py test_backend_*.py test_database.py test_server_*.py test_edgeql_ddl.py test_session.py\n\"\n run: |\n # Bump shmmax and shmall to avoid test failures.\n sudo sysctl -w kern.sysv.shmmax=12582912\n sudo sysctl -w kern.sysv.shmall=12582912\n edgedb-pkg/integration/macos/test.sh\n\n test-macos-aarch64:\n needs: [build-macos-aarch64]\n runs-on: ['macos-14']\n\n steps:\n - uses: actions/checkout@v4\n with:\n repository: edgedb/edgedb-pkg\n ref: master\n path: edgedb-pkg\n\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-macos-aarch64\n path: artifacts/macos-aarch64\n\n - name: Test\n env:\n PKG_PLATFORM: \"macos\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n run: |\n edgedb-pkg/integration/macos/test.sh\n collect:\n needs:\n - test-debian-buster-x86_64\n - test-debian-buster-aarch64\n - test-debian-bullseye-x86_64\n - test-debian-bullseye-aarch64\n - test-debian-bookworm-x86_64\n - test-debian-bookworm-aarch64\n - test-ubuntu-focal-x86_64\n - test-ubuntu-focal-aarch64\n - test-ubuntu-jammy-x86_64\n - test-ubuntu-jammy-aarch64\n - test-ubuntu-noble-x86_64\n - test-ubuntu-noble-aarch64\n - test-centos-8-x86_64\n - test-centos-8-aarch64\n - test-rockylinux-9-x86_64\n - test-rockylinux-9-aarch64\n - test-linux-x86_64\n - test-linux-aarch64\n - test-linuxmusl-x86_64\n - test-linuxmusl-aarch64\n - test-macos-x86_64\n - test-macos-aarch64\n runs-on: ubuntu-latest\n steps:\n - run: echo 'All build+tests passed, ready to publish now!'\n\n publish-debian-buster-x86_64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-buster-x86_64\n path: artifacts/debian-buster\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"buster\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-debian-buster-x86_64:\n needs: [publish-debian-buster-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-buster-x86_64\n path: artifacts/debian-buster\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: debian-buster\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-debian-buster:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"buster\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-debian-buster-aarch64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-buster-aarch64\n path: artifacts/debian-buster\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"buster\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-debian-buster-aarch64:\n needs: [publish-debian-buster-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-buster-aarch64\n path: artifacts/debian-buster\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: debian-buster\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-debian-buster:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"buster\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-debian-bullseye-x86_64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bullseye-x86_64\n path: artifacts/debian-bullseye\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bullseye\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-debian-bullseye-x86_64:\n needs: [publish-debian-bullseye-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bullseye-x86_64\n path: artifacts/debian-bullseye\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: debian-bullseye\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-debian-bullseye:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bullseye\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-debian-bullseye-aarch64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bullseye-aarch64\n path: artifacts/debian-bullseye\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bullseye\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-debian-bullseye-aarch64:\n needs: [publish-debian-bullseye-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bullseye-aarch64\n path: artifacts/debian-bullseye\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: debian-bullseye\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-debian-bullseye:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bullseye\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-debian-bookworm-x86_64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bookworm-x86_64\n path: artifacts/debian-bookworm\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bookworm\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-debian-bookworm-x86_64:\n needs: [publish-debian-bookworm-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bookworm-x86_64\n path: artifacts/debian-bookworm\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: debian-bookworm\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-debian-bookworm:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bookworm\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-debian-bookworm-aarch64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bookworm-aarch64\n path: artifacts/debian-bookworm\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bookworm\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-debian-bookworm-aarch64:\n needs: [publish-debian-bookworm-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bookworm-aarch64\n path: artifacts/debian-bookworm\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: debian-bookworm\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-debian-bookworm:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bookworm\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-ubuntu-focal-x86_64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-focal-x86_64\n path: artifacts/ubuntu-focal\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"focal\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-ubuntu-focal-x86_64:\n needs: [publish-ubuntu-focal-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-focal-x86_64\n path: artifacts/ubuntu-focal\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: ubuntu-focal\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-ubuntu-focal:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"focal\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-ubuntu-focal-aarch64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-focal-aarch64\n path: artifacts/ubuntu-focal\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"focal\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-ubuntu-focal-aarch64:\n needs: [publish-ubuntu-focal-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-focal-aarch64\n path: artifacts/ubuntu-focal\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: ubuntu-focal\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-ubuntu-focal:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"focal\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-ubuntu-jammy-x86_64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-jammy-x86_64\n path: artifacts/ubuntu-jammy\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"jammy\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-ubuntu-jammy-x86_64:\n needs: [publish-ubuntu-jammy-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-jammy-x86_64\n path: artifacts/ubuntu-jammy\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: ubuntu-jammy\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-ubuntu-jammy:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"jammy\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-ubuntu-jammy-aarch64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-jammy-aarch64\n path: artifacts/ubuntu-jammy\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"jammy\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-ubuntu-jammy-aarch64:\n needs: [publish-ubuntu-jammy-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-jammy-aarch64\n path: artifacts/ubuntu-jammy\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: ubuntu-jammy\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-ubuntu-jammy:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"jammy\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-ubuntu-noble-x86_64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-noble-x86_64\n path: artifacts/ubuntu-noble\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"noble\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-ubuntu-noble-x86_64:\n needs: [publish-ubuntu-noble-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-noble-x86_64\n path: artifacts/ubuntu-noble\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: ubuntu-noble\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-ubuntu-noble:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"noble\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-ubuntu-noble-aarch64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-noble-aarch64\n path: artifacts/ubuntu-noble\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"noble\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-ubuntu-noble-aarch64:\n needs: [publish-ubuntu-noble-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-noble-aarch64\n path: artifacts/ubuntu-noble\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: ubuntu-noble\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-ubuntu-noble:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"noble\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-centos-8-x86_64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-centos-8-x86_64\n path: artifacts/centos-8\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"centos\"\n PKG_PLATFORM_VERSION: \"8\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-centos-8-x86_64:\n needs: [publish-centos-8-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-centos-8-x86_64\n path: artifacts/centos-8\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: centos-8\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-centos-8:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"centos\"\n PKG_PLATFORM_VERSION: \"8\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-centos-8-aarch64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-centos-8-aarch64\n path: artifacts/centos-8\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"centos\"\n PKG_PLATFORM_VERSION: \"8\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-centos-8-aarch64:\n needs: [publish-centos-8-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-centos-8-aarch64\n path: artifacts/centos-8\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: centos-8\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-centos-8:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"centos\"\n PKG_PLATFORM_VERSION: \"8\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-rockylinux-9-x86_64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-rockylinux-9-x86_64\n path: artifacts/rockylinux-9\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"rockylinux\"\n PKG_PLATFORM_VERSION: \"9\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-rockylinux-9-x86_64:\n needs: [publish-rockylinux-9-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-rockylinux-9-x86_64\n path: artifacts/rockylinux-9\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: rockylinux-9\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-rockylinux-9:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"rockylinux\"\n PKG_PLATFORM_VERSION: \"9\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-rockylinux-9-aarch64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-rockylinux-9-aarch64\n path: artifacts/rockylinux-9\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"rockylinux\"\n PKG_PLATFORM_VERSION: \"9\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-rockylinux-9-aarch64:\n needs: [publish-rockylinux-9-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-rockylinux-9-aarch64\n path: artifacts/rockylinux-9\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: rockylinux-9\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-rockylinux-9:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"rockylinux\"\n PKG_PLATFORM_VERSION: \"9\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-linux-x86_64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linux-x86_64\n path: artifacts/linux-x86_64\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-linux-x86_64:\n needs: [publish-linux-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linux-x86_64\n path: artifacts/linux-x86_64\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: linux-x86_64\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-linux-x86_64:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-linux-aarch64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linux-aarch64\n path: artifacts/linux-aarch64\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-linux-aarch64:\n needs: [publish-linux-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linux-aarch64\n path: artifacts/linux-aarch64\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: linux-aarch64\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-linux-aarch64:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-linuxmusl-x86_64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linuxmusl-x86_64\n path: artifacts/linuxmusl-x86_64\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PKG_PLATFORM_LIBC: \"musl\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-linuxmusl-x86_64:\n needs: [publish-linuxmusl-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linuxmusl-x86_64\n path: artifacts/linuxmusl-x86_64\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: linuxmusl-x86_64\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-linuxmusl-x86_64:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-linuxmusl-aarch64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linuxmusl-aarch64\n path: artifacts/linuxmusl-aarch64\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PKG_PLATFORM_LIBC: \"musl\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-linuxmusl-aarch64:\n needs: [publish-linuxmusl-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linuxmusl-aarch64\n path: artifacts/linuxmusl-aarch64\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: linuxmusl-aarch64\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-linuxmusl-aarch64:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-macos-x86_64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-macos-x86_64\n path: artifacts/macos-x86_64\n\n - uses: actions/checkout@v4\n with:\n repository: edgedb/edgedb-pkg\n ref: master\n path: edgedb-pkg\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: macos-x86_64\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_PLATFORM: \"macos\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n publish-macos-aarch64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-macos-aarch64\n path: artifacts/macos-aarch64\n\n - uses: actions/checkout@v4\n with:\n repository: edgedb/edgedb-pkg\n ref: master\n path: edgedb-pkg\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: macos-aarch64\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_PLATFORM: \"macos\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n publish-docker:\n needs:\n - check-published-debian-bookworm-x86_64\n - check-published-debian-bookworm-aarch64\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/checkout@v4\n with:\n repository: geldata/gel-docker\n ref: master\n path: dockerfile\n\n - name: Login to Docker Hub\n uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0\n with:\n username: ${{ secrets.DOCKER_USERNAME }}\n password: ${{ secrets.DOCKER_PASSWORD }}\n\n - name: Login to GitHub Container Registry\n uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0\n with:\n registry: ghcr.io\n username: \"edgedb-ci\"\n password: ${{ secrets.GITHUB_CI_BOT_TOKEN }}\n\n - env:\n VERSION_SLOT: \"${{ needs.check-published-debian-bookworm-x86_64.outputs.version-slot }}\"\n VERSION_CORE: \"${{ needs.check-published-debian-bookworm-x86_64.outputs.version-core }}\"\n CATALOG_VERSION: \"${{ needs.check-published-debian-bookworm-x86_64.outputs.catalog-version }}\"\n PKG_SUBDIST: \"\"\n id: tags\n run: |\n set -e\n\n url='https://registry.hub.docker.com/v2/repositories/geldata/gel/tags?page_size=100'\n repo_tags=$(\n while [ -n \"$url\" ]; do\n resp=$(curl -L -s \"$url\")\n url=$(echo \"$resp\" | jq -r \".next\")\n if [ \"$url\" = \"null\" ] || [ -z \"$url\" ]; then\n break\n fi\n echo \"$resp\" | jq -r '.\"results\"[][\"name\"]'\n done | grep \"^[[:digit:]]\\+.*\" | grep -v \"alpha\\|beta\\|rc\" || :\n )\n\n tags=()\n\n if [ \"$PKG_SUBDIST\" = \"nightly\" ]; then\n tags+=(\n \"nightly\"\n \"nightly_${VERSION_SLOT}_cv${CATALOG_VERSION}\"\n )\n else\n tags+=( \"$VERSION_CORE\" )\n\n top=$(printf \"%s\\n%s\\n\" \"$VERSION_CORE\" \"$repo_tags\" \\\n | grep \"^${VERSION_SLOT}[\\.-]\" \\\n | sort --version-sort --reverse | head -n 1)\n if [ \"$top\" == \"$VERSION_CORE\" ]; then\n tags+=( \"$VERSION_SLOT\" )\n fi\n\n if [ -z \"$PKG_SUBDIST\" ]; then\n top=$(printf \"%s\\n%s\\n\" \"$VERSION_CORE\" \"$repo_tags\" \\\n | sort --version-sort --reverse | head -n 1)\n if [ \"$top\" == \"$VERSION_CORE\" ]; then\n tags+=( \"latest\" )\n fi\n fi\n fi\n\n fq_tags=()\n images=(\"geldata/gel\" \"ghcr.io/geldata/gel\")\n\n for image in \"${images[@]}\"; do\n fq_tags+=(\"${tags[@]/#/${image}:}\")\n done\n\n IFS=,\n echo \"tags=${fq_tags[*]}\" >> $GITHUB_OUTPUT\n\n - name: Set up QEMU\n uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0\n\n - name: Set up Docker Buildx\n uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0\n\n - name: Build and Publish Docker Image\n uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6.10.0\n with:\n push: true\n provenance: mode=max\n tags: \"${{ steps.tags.outputs.tags }}\"\n context: dockerfile\n build-args: |\n version=${{ needs.check-published-debian-bookworm-x86_64.outputs.version-slot }}\n exact_version=${{ needs.check-published-debian-bookworm-x86_64.outputs.version-core }}\n platforms: linux/amd64,linux/arm64\n\n workflow-notifications:\n if: failure() && github.event_name != 'pull_request'\n name: Notify in Slack on failures\n\n needs:\n - prep\n - collect\n - build-debian-buster-x86_64\n - test-debian-buster-x86_64\n - publish-debian-buster-x86_64\n - check-published-debian-buster-x86_64\n - build-debian-buster-aarch64\n - test-debian-buster-aarch64\n - publish-debian-buster-aarch64\n - check-published-debian-buster-aarch64\n - build-debian-bullseye-x86_64\n - test-debian-bullseye-x86_64\n - publish-debian-bullseye-x86_64\n - check-published-debian-bullseye-x86_64\n - build-debian-bullseye-aarch64\n - test-debian-bullseye-aarch64\n - publish-debian-bullseye-aarch64\n - check-published-debian-bullseye-aarch64\n - build-debian-bookworm-x86_64\n - test-debian-bookworm-x86_64\n - publish-debian-bookworm-x86_64\n - check-published-debian-bookworm-x86_64\n - build-debian-bookworm-aarch64\n - test-debian-bookworm-aarch64\n - publish-debian-bookworm-aarch64\n - check-published-debian-bookworm-aarch64\n - build-ubuntu-focal-x86_64\n - test-ubuntu-focal-x86_64\n - publish-ubuntu-focal-x86_64\n - check-published-ubuntu-focal-x86_64\n - build-ubuntu-focal-aarch64\n - test-ubuntu-focal-aarch64\n - publish-ubuntu-focal-aarch64\n - check-published-ubuntu-focal-aarch64\n - build-ubuntu-jammy-x86_64\n - test-ubuntu-jammy-x86_64\n - publish-ubuntu-jammy-x86_64\n - check-published-ubuntu-jammy-x86_64\n - build-ubuntu-jammy-aarch64\n - test-ubuntu-jammy-aarch64\n - publish-ubuntu-jammy-aarch64\n - check-published-ubuntu-jammy-aarch64\n - build-ubuntu-noble-x86_64\n - test-ubuntu-noble-x86_64\n - publish-ubuntu-noble-x86_64\n - check-published-ubuntu-noble-x86_64\n - build-ubuntu-noble-aarch64\n - test-ubuntu-noble-aarch64\n - publish-ubuntu-noble-aarch64\n - check-published-ubuntu-noble-aarch64\n - build-centos-8-x86_64\n - test-centos-8-x86_64\n - publish-centos-8-x86_64\n - check-published-centos-8-x86_64\n - build-centos-8-aarch64\n - test-centos-8-aarch64\n - publish-centos-8-aarch64\n - check-published-centos-8-aarch64\n - build-rockylinux-9-x86_64\n - test-rockylinux-9-x86_64\n - publish-rockylinux-9-x86_64\n - check-published-rockylinux-9-x86_64\n - build-rockylinux-9-aarch64\n - test-rockylinux-9-aarch64\n - publish-rockylinux-9-aarch64\n - check-published-rockylinux-9-aarch64\n - build-linux-x86_64\n - test-linux-x86_64\n - publish-linux-x86_64\n - check-published-linux-x86_64\n - build-linux-aarch64\n - test-linux-aarch64\n - publish-linux-aarch64\n - check-published-linux-aarch64\n - build-linuxmusl-x86_64\n - test-linuxmusl-x86_64\n - publish-linuxmusl-x86_64\n - check-published-linuxmusl-x86_64\n - build-linuxmusl-aarch64\n - test-linuxmusl-aarch64\n - publish-linuxmusl-aarch64\n - check-published-linuxmusl-aarch64\n - build-macos-x86_64\n - test-macos-x86_64\n - publish-macos-x86_64\n - build-macos-aarch64\n - test-macos-aarch64\n - publish-macos-aarch64\n - publish-docker\n runs-on: ubuntu-latest\n permissions:\n actions: 'read'\n steps:\n - name: Slack Workflow Notification\n uses: Gamesight/slack-workflow-status@26a36836c887f260477432e4314ec3490a84f309\n with:\n repo_token: ${{secrets.GITHUB_TOKEN}}\n slack_webhook_url: ${{secrets.ACTIONS_SLACK_WEBHOOK_URL}}\n name: 'Workflow notifications'\n icon_emoji: ':hammer:'\n include_jobs: 'on-failure'\n" }, { "path": ".github/workflows/build.testing.yml", "content": "name: Build Test and Publish a Testing Release\n\non:\n workflow_dispatch:\n inputs:\n gelpkg_ref:\n description: \"gel-pkg git ref used to build the packages\"\n default: \"master\"\n metapkg_ref:\n description: \"metapkg git ref used to build the packages\"\n default: \"master\"\n\njobs:\n prep:\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/checkout@v4\n\n\n\n build-debian-buster-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-debian-buster:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"buster\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-debian-buster-x86_64\n path: artifacts/debian-buster\n\n build-debian-buster-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-debian-buster:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"buster\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-debian-buster-aarch64\n path: artifacts/debian-buster\n\n build-debian-bullseye-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-debian-bullseye:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bullseye\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-debian-bullseye-x86_64\n path: artifacts/debian-bullseye\n\n build-debian-bullseye-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-debian-bullseye:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bullseye\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-debian-bullseye-aarch64\n path: artifacts/debian-bullseye\n\n build-debian-bookworm-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-debian-bookworm:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bookworm\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-debian-bookworm-x86_64\n path: artifacts/debian-bookworm\n\n build-debian-bookworm-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-debian-bookworm:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bookworm\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-debian-bookworm-aarch64\n path: artifacts/debian-bookworm\n\n build-ubuntu-focal-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-ubuntu-focal:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"focal\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-ubuntu-focal-x86_64\n path: artifacts/ubuntu-focal\n\n build-ubuntu-focal-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-ubuntu-focal:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"focal\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-ubuntu-focal-aarch64\n path: artifacts/ubuntu-focal\n\n build-ubuntu-jammy-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-ubuntu-jammy:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"jammy\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-ubuntu-jammy-x86_64\n path: artifacts/ubuntu-jammy\n\n build-ubuntu-jammy-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-ubuntu-jammy:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"jammy\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-ubuntu-jammy-aarch64\n path: artifacts/ubuntu-jammy\n\n build-ubuntu-noble-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-ubuntu-noble:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"noble\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-ubuntu-noble-x86_64\n path: artifacts/ubuntu-noble\n\n build-ubuntu-noble-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-ubuntu-noble:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"noble\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-ubuntu-noble-aarch64\n path: artifacts/ubuntu-noble\n\n build-centos-8-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-centos-8:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"centos\"\n PKG_PLATFORM_VERSION: \"8\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-centos-8-x86_64\n path: artifacts/centos-8\n\n build-centos-8-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-centos-8:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"centos\"\n PKG_PLATFORM_VERSION: \"8\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-centos-8-aarch64\n path: artifacts/centos-8\n\n build-rockylinux-9-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-rockylinux-9:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"rockylinux\"\n PKG_PLATFORM_VERSION: \"9\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-rockylinux-9-x86_64\n path: artifacts/rockylinux-9\n\n build-rockylinux-9-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-rockylinux-9:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"rockylinux\"\n PKG_PLATFORM_VERSION: \"9\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-rockylinux-9-aarch64\n path: artifacts/rockylinux-9\n\n build-linux-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-linux-x86_64:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n BUILD_GENERIC: true\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-linux-x86_64\n path: artifacts/linux-x86_64\n\n build-linux-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-linux-aarch64:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n BUILD_GENERIC: true\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-linux-aarch64\n path: artifacts/linux-aarch64\n\n build-linuxmusl-x86_64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-linuxmusl-x86_64:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n BUILD_GENERIC: true\n PKG_PLATFORM_LIBC: \"musl\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-linuxmusl-x86_64\n path: artifacts/linuxmusl-x86_64\n\n build-linuxmusl-aarch64:\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n needs: prep\n\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-linuxmusl-aarch64:latest\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n EXTRA_OPTIMIZATIONS: \"true\"\n BUILD_IS_RELEASE: \"true\"\n BUILD_GENERIC: true\n PKG_PLATFORM_LIBC: \"musl\"\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-linuxmusl-aarch64\n path: artifacts/linuxmusl-aarch64\n\n build-macos-x86_64:\n runs-on: ['macos-13']\n needs: prep\n\n\n steps:\n - name: Update Homebrew before installing Rust toolchain\n run: |\n # Homebrew renamed `rustup-init` to `rustup`:\n # https://github.com/Homebrew/homebrew-core/pull/177840\n # But the GitHub Action runner is not updated with this change yet.\n # This caused the later `brew update` in step `Build` to relink Rust\n # toolchain executables, overwriting the custom toolchain installed by\n # `dsherret/rust-toolchain-file`. So let's just run `brew update` early.\n brew update\n\n - uses: actions/checkout@v4\n if: true\n with:\n sparse-checkout: |\n rust-toolchain.toml\n sparse-checkout-cone-mode: false\n\n - name: Install Rust toolchain\n uses: dsherret/rust-toolchain-file@v1\n if: true\n\n - uses: actions/checkout@v4\n with:\n repository: edgedb/edgedb-pkg\n ref: master\n path: edgedb-pkg\n\n - name: Set up Python\n uses: actions/setup-python@v5\n if: true\n with:\n python-version: \"3.12\"\n\n - name: Set up NodeJS\n uses: actions/setup-node@v4\n if: true\n with:\n node-version: '20'\n\n - name: Install dependencies\n if: true\n run: |\n env HOMEBREW_NO_AUTO_UPDATE=1 brew install libmagic\n\n - name: Install an alias\n # This is probably not strictly needed, but sentencepiece build script reports\n # errors without it.\n if: true\n run: |\n printf '#!/bin/sh\\n\\nexec sysctl -n hw.logicalcpu' > /usr/local/bin/nproc\n chmod +x /usr/local/bin/nproc\n\n - name: Build\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n BUILD_IS_RELEASE: \"true\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"macos\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PKG_PLATFORM_ARCH: \"x86_64\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n BUILD_GENERIC: true\n CMAKE_POLICY_VERSION_MINIMUM: '3.5'\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n run: |\n edgedb-pkg/integration/macos/build.sh\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-macos-x86_64\n path: artifacts/macos-x86_64\n\n build-macos-aarch64:\n runs-on: ['macos-14']\n needs: prep\n\n\n steps:\n - name: Update Homebrew before installing Rust toolchain\n run: |\n # Homebrew renamed `rustup-init` to `rustup`:\n # https://github.com/Homebrew/homebrew-core/pull/177840\n # But the GitHub Action runner is not updated with this change yet.\n # This caused the later `brew update` in step `Build` to relink Rust\n # toolchain executables, overwriting the custom toolchain installed by\n # `dsherret/rust-toolchain-file`. So let's just run `brew update` early.\n brew update\n\n - uses: actions/checkout@v4\n if: true\n with:\n sparse-checkout: |\n rust-toolchain.toml\n sparse-checkout-cone-mode: false\n\n - name: Install Rust toolchain\n uses: dsherret/rust-toolchain-file@v1\n if: true\n\n - uses: actions/checkout@v4\n with:\n repository: edgedb/edgedb-pkg\n ref: master\n path: edgedb-pkg\n\n - name: Set up Python\n uses: actions/setup-python@v5\n if: true\n with:\n python-version: \"3.12\"\n\n - name: Set up NodeJS\n uses: actions/setup-node@v4\n if: true\n with:\n node-version: '20'\n\n - name: Install dependencies\n if: true\n run: |\n env HOMEBREW_NO_AUTO_UPDATE=1 brew install libmagic\n\n - name: Install an alias\n # This is probably not strictly needed, but sentencepiece build script reports\n # errors without it.\n if: true\n run: |\n printf '#!/bin/sh\\n\\nexec sysctl -n hw.logicalcpu' > /usr/local/bin/nproc\n chmod +x /usr/local/bin/nproc\n\n - name: Build\n env:\n PACKAGE: \"edgedbpkg.edgedb:Gel\"\n SRC_REF: \"${{ github.sha }}\"\n BUILD_IS_RELEASE: \"true\"\n PKG_REVISION: \"\"\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"macos\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PKG_PLATFORM_ARCH: \"aarch64\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n BUILD_GENERIC: true\n CMAKE_POLICY_VERSION_MINIMUM: '3.5'\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n run: |\n edgedb-pkg/integration/macos/build.sh\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-macos-aarch64\n path: artifacts/macos-aarch64\n\n test-debian-buster-x86_64:\n needs: [build-debian-buster-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-buster-x86_64\n path: artifacts/debian-buster\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-debian-buster:latest\n env:\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"buster\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-debian-buster-aarch64:\n needs: [build-debian-buster-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-buster-aarch64\n path: artifacts/debian-buster\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-debian-buster:latest\n env:\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"buster\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-debian-bullseye-x86_64:\n needs: [build-debian-bullseye-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bullseye-x86_64\n path: artifacts/debian-bullseye\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-debian-bullseye:latest\n env:\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bullseye\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-debian-bullseye-aarch64:\n needs: [build-debian-bullseye-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bullseye-aarch64\n path: artifacts/debian-bullseye\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-debian-bullseye:latest\n env:\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bullseye\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-debian-bookworm-x86_64:\n needs: [build-debian-bookworm-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bookworm-x86_64\n path: artifacts/debian-bookworm\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-debian-bookworm:latest\n env:\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bookworm\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-debian-bookworm-aarch64:\n needs: [build-debian-bookworm-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bookworm-aarch64\n path: artifacts/debian-bookworm\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-debian-bookworm:latest\n env:\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bookworm\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-ubuntu-focal-x86_64:\n needs: [build-ubuntu-focal-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-focal-x86_64\n path: artifacts/ubuntu-focal\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-ubuntu-focal:latest\n env:\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"focal\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-ubuntu-focal-aarch64:\n needs: [build-ubuntu-focal-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-focal-aarch64\n path: artifacts/ubuntu-focal\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-ubuntu-focal:latest\n env:\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"focal\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-ubuntu-jammy-x86_64:\n needs: [build-ubuntu-jammy-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-jammy-x86_64\n path: artifacts/ubuntu-jammy\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-ubuntu-jammy:latest\n env:\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"jammy\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-ubuntu-jammy-aarch64:\n needs: [build-ubuntu-jammy-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-jammy-aarch64\n path: artifacts/ubuntu-jammy\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-ubuntu-jammy:latest\n env:\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"jammy\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-ubuntu-noble-x86_64:\n needs: [build-ubuntu-noble-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-noble-x86_64\n path: artifacts/ubuntu-noble\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-ubuntu-noble:latest\n env:\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"noble\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-ubuntu-noble-aarch64:\n needs: [build-ubuntu-noble-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-noble-aarch64\n path: artifacts/ubuntu-noble\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-ubuntu-noble:latest\n env:\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"noble\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-centos-8-x86_64:\n needs: [build-centos-8-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-centos-8-x86_64\n path: artifacts/centos-8\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-centos-8:latest\n env:\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"centos\"\n PKG_PLATFORM_VERSION: \"8\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-centos-8-aarch64:\n needs: [build-centos-8-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-centos-8-aarch64\n path: artifacts/centos-8\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-centos-8:latest\n env:\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"centos\"\n PKG_PLATFORM_VERSION: \"8\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-rockylinux-9-x86_64:\n needs: [build-rockylinux-9-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-rockylinux-9-x86_64\n path: artifacts/rockylinux-9\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-rockylinux-9:latest\n env:\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"rockylinux\"\n PKG_PLATFORM_VERSION: \"9\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-rockylinux-9-aarch64:\n needs: [build-rockylinux-9-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-rockylinux-9-aarch64\n path: artifacts/rockylinux-9\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-rockylinux-9:latest\n env:\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"rockylinux\"\n PKG_PLATFORM_VERSION: \"9\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-linux-x86_64:\n needs: [build-linux-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linux-x86_64\n path: artifacts/linux-x86_64\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-linux-aarch64:\n needs: [build-linux-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linux-aarch64\n path: artifacts/linux-aarch64\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-linux-aarch64:latest\n env:\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PKG_PLATFORM_LIBC: \"\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-linuxmusl-x86_64:\n needs: [build-linuxmusl-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linuxmusl-x86_64\n path: artifacts/linuxmusl-x86_64\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-linuxmusl-x86_64:latest\n env:\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PKG_PLATFORM_LIBC: \"musl\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-linuxmusl-aarch64:\n needs: [build-linuxmusl-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linuxmusl-aarch64\n path: artifacts/linuxmusl-aarch64\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-linuxmusl-aarch64:latest\n env:\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PKG_PLATFORM_LIBC: \"musl\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: 0\n\n test-macos-x86_64:\n needs: [build-macos-x86_64]\n runs-on: ['macos-13']\n\n steps:\n - uses: actions/checkout@v4\n with:\n repository: edgedb/edgedb-pkg\n ref: master\n path: edgedb-pkg\n\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-macos-x86_64\n path: artifacts/macos-x86_64\n\n - name: Test\n env:\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"macos\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" test_dump*.py test_backend_*.py test_database.py test_server_*.py test_edgeql_ddl.py test_session.py\n\"\n run: |\n # Bump shmmax and shmall to avoid test failures.\n sudo sysctl -w kern.sysv.shmmax=12582912\n sudo sysctl -w kern.sysv.shmall=12582912\n edgedb-pkg/integration/macos/test.sh\n\n test-macos-aarch64:\n needs: [build-macos-aarch64]\n runs-on: ['macos-14']\n\n steps:\n - uses: actions/checkout@v4\n with:\n repository: edgedb/edgedb-pkg\n ref: master\n path: edgedb-pkg\n\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-macos-aarch64\n path: artifacts/macos-aarch64\n\n - name: Test\n env:\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"macos\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PKG_TEST_SELECT: \"\"\n PKG_TEST_EXCLUDE: \"\"\n PKG_TEST_FILES: \" \"\n run: |\n edgedb-pkg/integration/macos/test.sh\n collect:\n needs:\n - test-debian-buster-x86_64\n - test-debian-buster-aarch64\n - test-debian-bullseye-x86_64\n - test-debian-bullseye-aarch64\n - test-debian-bookworm-x86_64\n - test-debian-bookworm-aarch64\n - test-ubuntu-focal-x86_64\n - test-ubuntu-focal-aarch64\n - test-ubuntu-jammy-x86_64\n - test-ubuntu-jammy-aarch64\n - test-ubuntu-noble-x86_64\n - test-ubuntu-noble-aarch64\n - test-centos-8-x86_64\n - test-centos-8-aarch64\n - test-rockylinux-9-x86_64\n - test-rockylinux-9-aarch64\n - test-linux-x86_64\n - test-linux-aarch64\n - test-linuxmusl-x86_64\n - test-linuxmusl-aarch64\n - test-macos-x86_64\n - test-macos-aarch64\n runs-on: ubuntu-latest\n steps:\n - run: echo 'All build+tests passed, ready to publish now!'\n\n publish-debian-buster-x86_64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-buster-x86_64\n path: artifacts/debian-buster\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"buster\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-debian-buster-x86_64:\n needs: [publish-debian-buster-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-buster-x86_64\n path: artifacts/debian-buster\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: debian-buster\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-debian-buster:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"buster\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-debian-buster-aarch64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-buster-aarch64\n path: artifacts/debian-buster\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"buster\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-debian-buster-aarch64:\n needs: [publish-debian-buster-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-buster-aarch64\n path: artifacts/debian-buster\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: debian-buster\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-debian-buster:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"buster\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-debian-bullseye-x86_64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bullseye-x86_64\n path: artifacts/debian-bullseye\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bullseye\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-debian-bullseye-x86_64:\n needs: [publish-debian-bullseye-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bullseye-x86_64\n path: artifacts/debian-bullseye\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: debian-bullseye\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-debian-bullseye:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bullseye\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-debian-bullseye-aarch64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bullseye-aarch64\n path: artifacts/debian-bullseye\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bullseye\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-debian-bullseye-aarch64:\n needs: [publish-debian-bullseye-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bullseye-aarch64\n path: artifacts/debian-bullseye\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: debian-bullseye\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-debian-bullseye:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bullseye\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-debian-bookworm-x86_64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bookworm-x86_64\n path: artifacts/debian-bookworm\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bookworm\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-debian-bookworm-x86_64:\n needs: [publish-debian-bookworm-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bookworm-x86_64\n path: artifacts/debian-bookworm\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: debian-bookworm\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-debian-bookworm:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bookworm\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-debian-bookworm-aarch64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bookworm-aarch64\n path: artifacts/debian-bookworm\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bookworm\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-debian-bookworm-aarch64:\n needs: [publish-debian-bookworm-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-debian-bookworm-aarch64\n path: artifacts/debian-bookworm\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: debian-bookworm\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-debian-bookworm:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"debian\"\n PKG_PLATFORM_VERSION: \"bookworm\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-ubuntu-focal-x86_64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-focal-x86_64\n path: artifacts/ubuntu-focal\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"focal\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-ubuntu-focal-x86_64:\n needs: [publish-ubuntu-focal-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-focal-x86_64\n path: artifacts/ubuntu-focal\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: ubuntu-focal\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-ubuntu-focal:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"focal\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-ubuntu-focal-aarch64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-focal-aarch64\n path: artifacts/ubuntu-focal\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"focal\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-ubuntu-focal-aarch64:\n needs: [publish-ubuntu-focal-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-focal-aarch64\n path: artifacts/ubuntu-focal\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: ubuntu-focal\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-ubuntu-focal:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"focal\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-ubuntu-jammy-x86_64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-jammy-x86_64\n path: artifacts/ubuntu-jammy\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"jammy\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-ubuntu-jammy-x86_64:\n needs: [publish-ubuntu-jammy-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-jammy-x86_64\n path: artifacts/ubuntu-jammy\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: ubuntu-jammy\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-ubuntu-jammy:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"jammy\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-ubuntu-jammy-aarch64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-jammy-aarch64\n path: artifacts/ubuntu-jammy\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"jammy\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-ubuntu-jammy-aarch64:\n needs: [publish-ubuntu-jammy-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-jammy-aarch64\n path: artifacts/ubuntu-jammy\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: ubuntu-jammy\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-ubuntu-jammy:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"jammy\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-ubuntu-noble-x86_64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-noble-x86_64\n path: artifacts/ubuntu-noble\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"noble\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-ubuntu-noble-x86_64:\n needs: [publish-ubuntu-noble-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-noble-x86_64\n path: artifacts/ubuntu-noble\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: ubuntu-noble\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-ubuntu-noble:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"noble\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-ubuntu-noble-aarch64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-noble-aarch64\n path: artifacts/ubuntu-noble\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"noble\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-ubuntu-noble-aarch64:\n needs: [publish-ubuntu-noble-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-ubuntu-noble-aarch64\n path: artifacts/ubuntu-noble\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: ubuntu-noble\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-ubuntu-noble:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"ubuntu\"\n PKG_PLATFORM_VERSION: \"noble\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-centos-8-x86_64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-centos-8-x86_64\n path: artifacts/centos-8\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"centos\"\n PKG_PLATFORM_VERSION: \"8\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-centos-8-x86_64:\n needs: [publish-centos-8-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-centos-8-x86_64\n path: artifacts/centos-8\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: centos-8\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-centos-8:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"centos\"\n PKG_PLATFORM_VERSION: \"8\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-centos-8-aarch64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-centos-8-aarch64\n path: artifacts/centos-8\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"centos\"\n PKG_PLATFORM_VERSION: \"8\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-centos-8-aarch64:\n needs: [publish-centos-8-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-centos-8-aarch64\n path: artifacts/centos-8\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: centos-8\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-centos-8:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"centos\"\n PKG_PLATFORM_VERSION: \"8\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-rockylinux-9-x86_64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-rockylinux-9-x86_64\n path: artifacts/rockylinux-9\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"rockylinux\"\n PKG_PLATFORM_VERSION: \"9\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-rockylinux-9-x86_64:\n needs: [publish-rockylinux-9-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-rockylinux-9-x86_64\n path: artifacts/rockylinux-9\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: rockylinux-9\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-rockylinux-9:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"rockylinux\"\n PKG_PLATFORM_VERSION: \"9\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-rockylinux-9-aarch64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-rockylinux-9-aarch64\n path: artifacts/rockylinux-9\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"rockylinux\"\n PKG_PLATFORM_VERSION: \"9\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-rockylinux-9-aarch64:\n needs: [publish-rockylinux-9-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-rockylinux-9-aarch64\n path: artifacts/rockylinux-9\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: rockylinux-9\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-rockylinux-9:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"rockylinux\"\n PKG_PLATFORM_VERSION: \"9\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-linux-x86_64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linux-x86_64\n path: artifacts/linux-x86_64\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-linux-x86_64:\n needs: [publish-linux-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linux-x86_64\n path: artifacts/linux-x86_64\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: linux-x86_64\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-linux-x86_64:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-linux-aarch64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linux-aarch64\n path: artifacts/linux-aarch64\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PKG_PLATFORM_LIBC: \"\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-linux-aarch64:\n needs: [publish-linux-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linux-aarch64\n path: artifacts/linux-aarch64\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: linux-aarch64\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-linux-aarch64:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-linuxmusl-x86_64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linuxmusl-x86_64\n path: artifacts/linuxmusl-x86_64\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PKG_PLATFORM_LIBC: \"musl\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-linuxmusl-x86_64:\n needs: [publish-linuxmusl-x86_64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'x64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linuxmusl-x86_64\n path: artifacts/linuxmusl-x86_64\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: linuxmusl-x86_64\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-linuxmusl-x86_64:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-linuxmusl-aarch64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linuxmusl-aarch64\n path: artifacts/linuxmusl-aarch64\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PKG_PLATFORM_LIBC: \"musl\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published-linuxmusl-aarch64:\n needs: [publish-linuxmusl-aarch64]\n runs-on: ['package-builder', 'self-hosted', 'linux', 'arm64']\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-linuxmusl-aarch64\n path: artifacts/linuxmusl-aarch64\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: linuxmusl-aarch64\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-linuxmusl-aarch64:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n PKG_SUBDIST: \"testing\"\n PACKAGE_SERVER: sftp://uploader@package-upload.edgedb.net:22/\n PKG_PLATFORM: \"linux\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n\n publish-macos-x86_64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-macos-x86_64\n path: artifacts/macos-x86_64\n\n - uses: actions/checkout@v4\n with:\n repository: edgedb/edgedb-pkg\n ref: master\n path: edgedb-pkg\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: macos-x86_64\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"macos\"\n PKG_PLATFORM_VERSION: \"x86_64\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n publish-macos-aarch64:\n needs: [collect]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-macos-aarch64\n path: artifacts/macos-aarch64\n\n - uses: actions/checkout@v4\n with:\n repository: edgedb/edgedb-pkg\n ref: master\n path: edgedb-pkg\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: macos-aarch64\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n PKG_SUBDIST: \"testing\"\n PKG_PLATFORM: \"macos\"\n PKG_PLATFORM_VERSION: \"aarch64\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n publish-docker:\n needs:\n - check-published-debian-bookworm-x86_64\n - check-published-debian-bookworm-aarch64\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/checkout@v4\n with:\n repository: geldata/gel-docker\n ref: master\n path: dockerfile\n\n - name: Login to Docker Hub\n uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0\n with:\n username: ${{ secrets.DOCKER_USERNAME }}\n password: ${{ secrets.DOCKER_PASSWORD }}\n\n - name: Login to GitHub Container Registry\n uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0\n with:\n registry: ghcr.io\n username: \"edgedb-ci\"\n password: ${{ secrets.GITHUB_CI_BOT_TOKEN }}\n\n - env:\n VERSION_SLOT: \"${{ needs.check-published-debian-bookworm-x86_64.outputs.version-slot }}\"\n VERSION_CORE: \"${{ needs.check-published-debian-bookworm-x86_64.outputs.version-core }}\"\n CATALOG_VERSION: \"${{ needs.check-published-debian-bookworm-x86_64.outputs.catalog-version }}\"\n PKG_SUBDIST: \"testing\"\n id: tags\n run: |\n set -e\n\n url='https://registry.hub.docker.com/v2/repositories/geldata/gel/tags?page_size=100'\n repo_tags=$(\n while [ -n \"$url\" ]; do\n resp=$(curl -L -s \"$url\")\n url=$(echo \"$resp\" | jq -r \".next\")\n if [ \"$url\" = \"null\" ] || [ -z \"$url\" ]; then\n break\n fi\n echo \"$resp\" | jq -r '.\"results\"[][\"name\"]'\n done | grep \"^[[:digit:]]\\+.*\" | grep -v \"alpha\\|beta\\|rc\" || :\n )\n\n tags=()\n\n if [ \"$PKG_SUBDIST\" = \"nightly\" ]; then\n tags+=(\n \"nightly\"\n \"nightly_${VERSION_SLOT}_cv${CATALOG_VERSION}\"\n )\n else\n tags+=( \"$VERSION_CORE\" )\n\n top=$(printf \"%s\\n%s\\n\" \"$VERSION_CORE\" \"$repo_tags\" \\\n | grep \"^${VERSION_SLOT}[\\.-]\" \\\n | sort --version-sort --reverse | head -n 1)\n if [ \"$top\" == \"$VERSION_CORE\" ]; then\n tags+=( \"$VERSION_SLOT\" )\n fi\n\n if [ -z \"$PKG_SUBDIST\" ]; then\n top=$(printf \"%s\\n%s\\n\" \"$VERSION_CORE\" \"$repo_tags\" \\\n | sort --version-sort --reverse | head -n 1)\n if [ \"$top\" == \"$VERSION_CORE\" ]; then\n tags+=( \"latest\" )\n fi\n fi\n fi\n\n fq_tags=()\n images=(\"geldata/gel\" \"ghcr.io/geldata/gel\")\n\n for image in \"${images[@]}\"; do\n fq_tags+=(\"${tags[@]/#/${image}:}\")\n done\n\n IFS=,\n echo \"tags=${fq_tags[*]}\" >> $GITHUB_OUTPUT\n\n - name: Set up QEMU\n uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0\n\n - name: Set up Docker Buildx\n uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0\n\n - name: Build and Publish Docker Image\n uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6.10.0\n with:\n push: true\n provenance: mode=max\n tags: \"${{ steps.tags.outputs.tags }}\"\n context: dockerfile\n build-args: |\n version=${{ needs.check-published-debian-bookworm-x86_64.outputs.version-slot }}\n exact_version=${{ needs.check-published-debian-bookworm-x86_64.outputs.version-core }}\n subdist=testing\n platforms: linux/amd64,linux/arm64\n\n workflow-notifications:\n if: failure() && github.event_name != 'pull_request'\n name: Notify in Slack on failures\n\n needs:\n - prep\n - collect\n - build-debian-buster-x86_64\n - test-debian-buster-x86_64\n - publish-debian-buster-x86_64\n - check-published-debian-buster-x86_64\n - build-debian-buster-aarch64\n - test-debian-buster-aarch64\n - publish-debian-buster-aarch64\n - check-published-debian-buster-aarch64\n - build-debian-bullseye-x86_64\n - test-debian-bullseye-x86_64\n - publish-debian-bullseye-x86_64\n - check-published-debian-bullseye-x86_64\n - build-debian-bullseye-aarch64\n - test-debian-bullseye-aarch64\n - publish-debian-bullseye-aarch64\n - check-published-debian-bullseye-aarch64\n - build-debian-bookworm-x86_64\n - test-debian-bookworm-x86_64\n - publish-debian-bookworm-x86_64\n - check-published-debian-bookworm-x86_64\n - build-debian-bookworm-aarch64\n - test-debian-bookworm-aarch64\n - publish-debian-bookworm-aarch64\n - check-published-debian-bookworm-aarch64\n - build-ubuntu-focal-x86_64\n - test-ubuntu-focal-x86_64\n - publish-ubuntu-focal-x86_64\n - check-published-ubuntu-focal-x86_64\n - build-ubuntu-focal-aarch64\n - test-ubuntu-focal-aarch64\n - publish-ubuntu-focal-aarch64\n - check-published-ubuntu-focal-aarch64\n - build-ubuntu-jammy-x86_64\n - test-ubuntu-jammy-x86_64\n - publish-ubuntu-jammy-x86_64\n - check-published-ubuntu-jammy-x86_64\n - build-ubuntu-jammy-aarch64\n - test-ubuntu-jammy-aarch64\n - publish-ubuntu-jammy-aarch64\n - check-published-ubuntu-jammy-aarch64\n - build-ubuntu-noble-x86_64\n - test-ubuntu-noble-x86_64\n - publish-ubuntu-noble-x86_64\n - check-published-ubuntu-noble-x86_64\n - build-ubuntu-noble-aarch64\n - test-ubuntu-noble-aarch64\n - publish-ubuntu-noble-aarch64\n - check-published-ubuntu-noble-aarch64\n - build-centos-8-x86_64\n - test-centos-8-x86_64\n - publish-centos-8-x86_64\n - check-published-centos-8-x86_64\n - build-centos-8-aarch64\n - test-centos-8-aarch64\n - publish-centos-8-aarch64\n - check-published-centos-8-aarch64\n - build-rockylinux-9-x86_64\n - test-rockylinux-9-x86_64\n - publish-rockylinux-9-x86_64\n - check-published-rockylinux-9-x86_64\n - build-rockylinux-9-aarch64\n - test-rockylinux-9-aarch64\n - publish-rockylinux-9-aarch64\n - check-published-rockylinux-9-aarch64\n - build-linux-x86_64\n - test-linux-x86_64\n - publish-linux-x86_64\n - check-published-linux-x86_64\n - build-linux-aarch64\n - test-linux-aarch64\n - publish-linux-aarch64\n - check-published-linux-aarch64\n - build-linuxmusl-x86_64\n - test-linuxmusl-x86_64\n - publish-linuxmusl-x86_64\n - check-published-linuxmusl-x86_64\n - build-linuxmusl-aarch64\n - test-linuxmusl-aarch64\n - publish-linuxmusl-aarch64\n - check-published-linuxmusl-aarch64\n - build-macos-x86_64\n - test-macos-x86_64\n - publish-macos-x86_64\n - build-macos-aarch64\n - test-macos-aarch64\n - publish-macos-aarch64\n - publish-docker\n runs-on: ubuntu-latest\n permissions:\n actions: 'read'\n steps:\n - name: Slack Workflow Notification\n uses: Gamesight/slack-workflow-status@26a36836c887f260477432e4314ec3490a84f309\n with:\n repo_token: ${{secrets.GITHUB_TOKEN}}\n slack_webhook_url: ${{secrets.ACTIONS_SLACK_WEBHOOK_URL}}\n name: 'Workflow notifications'\n icon_emoji: ':hammer:'\n include_jobs: 'on-failure'\n" }, { "path": ".github/workflows/docs-preview-deploy.yml", "content": "name: Docs Preview Deploy\n\non:\n pull_request:\n paths:\n - \"docs/**\"\n\njobs:\n deploy:\n runs-on: ubuntu-latest\n permissions: write-all\n steps:\n - uses: actions/checkout@v4\n - uses: actions/github-script@v7\n env:\n VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }}\n VERCEL_TEAM_ID: ${{ secrets.VERCEL_TEAM_ID }}\n with:\n script: |\n const script = require('./.github/scripts/docs/preview-deploy.js');\n await script({github, context});\n" }, { "path": ".github/workflows/docs.yml", "content": "name: Deploy Documentation Changes\n\non:\n push:\n branches:\n - master\n - release/**\n paths:\n - \"docs/**\"\n workflow_dispatch:\n\njobs:\n deploy:\n runs-on: ubuntu-latest\n steps:\n - name: Trigger vercel deploy hook\n run: curl \\\n --fail-with-body \\\n --request POST \\\n ${{ secrets.VERCEL_DOC_DEPLOY_URL_HOOK }}\n" }, { "path": ".github/workflows/pull-request-meta.yml", "content": "name: Pull Request Meta\n\non:\n pull_request:\n types: [opened, edited, synchronize]\n\nconcurrency:\n group: ${{ github.workflow }}-${{ github.event.pull_request.number }}\n cancel-in-progress: true\n\njobs:\n\n test-pr:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - name: Verify that postgres/ was not changed unintentionally\n env:\n PR_TITLE: ${{ github.event.pull_request.title }}\n shell: bash\n run: |\n required_prefix=\"Update bundled PostgreSQL\"\n\n if [[ \"$PR_TITLE\" == $required_prefix* ]]; then\n exit 0\n fi\n\n if git diff --quiet \\\n ${{ github.event.pull_request.base.sha }} \\\n ${{ github.event.pull_request.head.sha }} -- postgres/\n then\n echo 'all ok'\n else\n echo \"postgres/ submodule has been changed,\"\\\n \"but PR title does not indicate that\"\n echo \"(it should start with '$required_prefix')\"\n exit 1\n fi\n" }, { "path": ".github/workflows/tests.ha.yml", "content": "name: High Availability Tests\n\non:\n workflow_dispatch:\n inputs: {}\n workflow_run:\n workflows: [\"Tests\"]\n types:\n - completed\n\njobs:\n build:\n runs-on: ubuntu-latest\n if: github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch'\n\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - uses: actions/checkout@v4\n with:\n fetch-depth: 50\n submodules: true\n\n - name: Set up Python\n uses: actions/setup-python@v5\n id: setup-python\n with:\n python-version: '3.12.2'\n cache: 'pip'\n cache-dependency-path: |\n pyproject.toml\n\n # The below is technically a lie as we are technically not\n # inside a virtual env, but there is really no reason to bother\n # actually creating and activating one as below works just fine.\n - name: Export $VIRTUAL_ENV\n run: |\n venv=\"$(python -c 'import sys; sys.stdout.write(sys.prefix)')\"\n echo \"VIRTUAL_ENV=${venv}\" >> $GITHUB_ENV\n\n - name: Set up uv cache\n uses: actions/cache@v4\n with:\n path: ~/.cache/uv\n key: uv-cache-${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}\n\n - name: Cached requirements.txt\n uses: actions/cache@v4\n id: requirements-cache\n with:\n path: requirements.txt\n key: edb-requirements-${{ hashFiles('pyproject.toml') }}\n\n - name: Compute requirements.txt\n if: steps.requirements-cache.outputs.cache-hit != 'true'\n run: |\n python -m pip install pip-tools\n pip-compile --no-strip-extras --all-build-deps \\\n --extra test,language-server \\\n --output-file requirements.txt pyproject.toml\n\n - name: Install Python dependencies\n run: |\n python -c \"import sys; print(sys.prefix)\"\n python -m pip install uv~=0.1.0 && uv pip install -U -r requirements.txt\n # 80.9.0 breaks our sphinx, and it keeps sneaking in\n uv pip install setuptools==80.8.0\n\n # Our HA tests currently only work on Postgres 14 (see #6332),\n # so check it out before we compute our build cache keys.\n - name: Switch back to Postgres 14\n shell: bash\n run: |\n set -e\n cd postgres\n # Fetch postgres 14, since the clone was shallow\n git fetch origin REL_14_8 --depth=1\n # For whatever reason the tag doesn't get fetched, so find it\n # at FETCH_HEAD\n git checkout FETCH_HEAD\n\n - name: Compute cache keys\n env:\n GIST_TOKEN: ${{ secrets.CI_BOT_GIST_TOKEN }}\n run: |\n mkdir -p shared-artifacts\n if [ \"$(uname)\" = \"Darwin\" ]; then\n find /usr/lib -type f -name 'lib*' -exec stat -f '%N %z' {} + | sort | shasum -a 256 | cut -d ' ' -f1 > shared-artifacts/lib_cache_key.txt\n else\n find /usr/lib -type f -name 'lib*' -printf '%P %s\\n' | sort | sha256sum | cut -d ' ' -f1 > shared-artifacts/lib_cache_key.txt\n fi\n python setup.py -q ci_helper --type rust >shared-artifacts/rust_cache_key.txt\n python setup.py -q ci_helper --type ext >shared-artifacts/ext_cache_key.txt\n python setup.py -q ci_helper --type parsers >shared-artifacts/parsers_cache_key.txt\n python setup.py -q ci_helper --type postgres >shared-artifacts/postgres_git_rev.txt\n python setup.py -q ci_helper --type libpg_query >shared-artifacts/libpg_query_git_rev.txt\n echo 'f8cd94309eaccbfba5dea7835b88c78377608a37' >shared-artifacts/stolon_git_rev.txt\n python setup.py -q ci_helper --type bootstrap >shared-artifacts/bootstrap_cache_key.txt\n echo POSTGRES_GIT_REV=$(cat shared-artifacts/postgres_git_rev.txt) >> $GITHUB_ENV\n echo LIBPG_QUERY_GIT_REV=$(cat shared-artifacts/libpg_query_git_rev.txt) >> $GITHUB_ENV\n echo STOLON_GIT_REV=$(cat shared-artifacts/stolon_git_rev.txt) >> $GITHUB_ENV\n echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV\n echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV\n\n - name: Upload shared artifacts\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: shared-artifacts\n path: shared-artifacts\n retention-days: 1\n\n # Restore binary cache\n\n - name: Handle cached Rust extensions\n uses: actions/cache@v4\n id: rust-cache\n with:\n path: build/rust_extensions\n key: edb-rust-v4-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n restore-keys: |\n edb-rust-v4-\n\n - name: Handle cached Cython extensions\n uses: actions/cache@v4\n id: ext-cache\n with:\n path: build/extensions\n key: edb-ext-v6-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Handle cached PostgreSQL build\n uses: actions/cache@v4\n id: postgres-cache\n with:\n path: build/postgres/install\n key: edb-postgres-v3-${{ env.POSTGRES_GIT_REV }}-${{ hashFiles('shared-artifacts/lib_cache_key.txt') }}\n\n - name: Handle cached Stolon build\n uses: actions/cache@v4\n id: stolon-cache\n with:\n path: build/stolon/bin\n key: edb-stolon-v2-${{ env.STOLON_GIT_REV }}\n\n - name: Handle cached libpg_query build\n uses: actions/cache@v4\n id: libpg-query-cache\n with:\n path: edb/pgsql/parser/libpg_query/libpg_query.a\n key: edb-libpg_query-v1-${{ env.LIBPG_QUERY_GIT_REV }}\n\n # Install system dependencies for building\n\n - name: Install system deps\n if: |\n steps.rust-cache.outputs.cache-hit != 'true' ||\n steps.ext-cache.outputs.cache-hit != 'true' ||\n steps.stolon-cache.outputs.cache-hit != 'true' ||\n steps.postgres-cache.outputs.cache-hit != 'true'\n run: |\n sudo apt-get update\n sudo apt-get install -y uuid-dev libreadline-dev bison flex libprotobuf-c-dev\n\n - name: Install Rust toolchain\n if: steps.rust-cache.outputs.cache-hit != 'true'\n uses: dsherret/rust-toolchain-file@v1\n\n # Build Rust extensions\n\n - name: Handle Rust extensions build cache\n uses: actions/cache@v4\n if: steps.rust-cache.outputs.cache-hit != 'true'\n with:\n path: ${{ env.BUILD_TEMP }}/rust/extensions\n key: edb-rust-build-v1-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n restore-keys: |\n edb-rust-build-v1-\n\n - name: Build Rust extensions\n env:\n CARGO_HOME: ${{ env.BUILD_TEMP }}/rust/extensions/cargo_home\n CACHE_HIT: ${{ steps.rust-cache.outputs.cache-hit }}\n run: |\n if [[ \"$CACHE_HIT\" != \"true\" ]]; then\n rm -rf ${BUILD_LIB}\n mkdir -p build/rust_extensions\n rsync -av ./build/rust_extensions/ ${BUILD_LIB}/\n python setup.py -v build_rust\n rsync -av ${BUILD_LIB}/ build/rust_extensions/\n rm -rf ${BUILD_LIB}\n fi\n rsync -av ./build/rust_extensions/edb/ ./edb/\n\n # Build libpg_query\n\n - name: Build libpg_query\n if: |\n steps.libpg-query-cache.outputs.cache-hit != 'true' &&\n steps.ext-cache.outputs.cache-hit != 'true'\n run: |\n python setup.py build_libpg_query\n\n # Build extensions\n\n - name: Handle Cython extensions build cache\n uses: actions/cache@v4\n if: steps.ext-cache.outputs.cache-hit != 'true'\n with:\n path: ${{ env.BUILD_TEMP }}/edb\n key: edb-ext-build-v4-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Build Cython extensions\n env:\n CACHE_HIT: ${{ steps.ext-cache.outputs.cache-hit }}\n BUILD_EXT_MODE: py-only\n run: |\n if [[ \"$CACHE_HIT\" != \"true\" ]]; then\n rm -rf ${BUILD_LIB}\n mkdir -p ./build/extensions\n rsync -av ./build/extensions/ ${BUILD_LIB}/\n BUILD_EXT_MODE=py-only python setup.py -v build_ext\n rsync -av ${BUILD_LIB}/ ./build/extensions/\n rm -rf ${BUILD_LIB}\n fi\n rsync -av ./build/extensions/edb/ ./edb/\n\n # Build parsers\n\n - name: Handle compiled parsers cache\n uses: actions/cache@v4\n id: parsers-cache\n with:\n path: build/lib\n key: edb-parsers-v3-${{ hashFiles('shared-artifacts/parsers_cache_key.txt') }}\n restore-keys: |\n edb-parsers-v3-\n\n - name: Build parsers\n env:\n CACHE_HIT: ${{ steps.parsers-cache.outputs.cache-hit }}\n run: |\n if [[ \"$CACHE_HIT\" != \"true\" ]]; then\n rm -rf ${BUILD_LIB}\n mkdir -p ./build/lib\n rsync -av ./build/lib/ ${BUILD_LIB}/\n python setup.py -v build_parsers\n rsync -av ${BUILD_LIB}/ ./build/lib/\n rm -rf ${BUILD_LIB}\n fi\n rsync -av ./build/lib/edb/ ./edb/\n\n # Build PostgreSQL\n\n - name: Build PostgreSQL\n env:\n CACHE_HIT: ${{ steps.postgres-cache.outputs.cache-hit }}\n run: |\n if [[ \"$CACHE_HIT\" == \"true\" ]]; then\n cp build/postgres/install/stamp build/postgres/\n else\n python setup.py build_postgres\n cp build/postgres/stamp build/postgres/install/\n fi\n\n # Build Stolon\n\n - name: Set up Go\n if: steps.stolon-cache.outputs.cache-hit != 'true'\n uses: actions/setup-go@v2\n with:\n go-version: 1.16\n\n - uses: actions/checkout@v4\n if: steps.stolon-cache.outputs.cache-hit != 'true'\n with:\n repository: edgedb/stolon\n path: build/stolon\n ref: ${{ env.STOLON_GIT_REV }}\n fetch-depth: 0\n submodules: false\n\n - name: Build Stolon\n if: steps.stolon-cache.outputs.cache-hit != 'true'\n run: |\n mkdir -p build/stolon/bin/\n curl -fsSL https://releases.hashicorp.com/consul/1.10.1/consul_1.10.1_linux_amd64.zip | zcat > build/stolon/bin/consul\n chmod +x build/stolon/bin/consul\n cd build/stolon && make\n\n # Install edgedb-server and populate egg-info\n\n - name: Install edgedb-server\n env:\n BUILD_EXT_MODE: skip\n run: |\n # --no-build-isolation because we have explicitly installed all deps\n # and don't want them to be reinstalled in an \"isolated env\".\n pip install --no-build-isolation --no-deps -e .[test,docs]\n\n # Refresh the bootstrap cache\n\n - name: Handle bootstrap cache\n uses: actions/cache@v4\n id: bootstrap-cache\n with:\n path: build/cache\n key: edb-bootstrap-v2-${{ hashFiles('shared-artifacts/bootstrap_cache_key.txt') }}\n restore-keys: |\n edb-bootstrap-v2-\n\n - name: Bootstrap EdgeDB Server\n if: steps.bootstrap-cache.outputs.cache-hit != 'true'\n run: |\n edb server --bootstrap-only\n\n ha-test:\n needs: build\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - uses: actions/checkout@v4\n with:\n fetch-depth: 50\n submodules: true\n\n - name: Set up Python\n uses: actions/setup-python@v5\n id: setup-python\n with:\n python-version: '3.12.2'\n cache: 'pip'\n cache-dependency-path: |\n pyproject.toml\n\n # The below is technically a lie as we are technically not\n # inside a virtual env, but there is really no reason to bother\n # actually creating and activating one as below works just fine.\n - name: Export $VIRTUAL_ENV\n run: |\n venv=\"$(python -c 'import sys; sys.stdout.write(sys.prefix)')\"\n echo \"VIRTUAL_ENV=${venv}\" >> $GITHUB_ENV\n\n - name: Set up uv cache\n uses: actions/cache@v4\n with:\n path: ~/.cache/uv\n key: uv-cache-${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}\n \n - name: Download requirements.txt\n uses: actions/cache@v4\n with:\n path: requirements.txt\n key: edb-requirements-${{ hashFiles('pyproject.toml') }}\n\n - name: Install Python dependencies\n run: |\n python -m pip install uv~=0.1.0 && uv pip install -U -r requirements.txt\n # 80.9.0 breaks our sphinx, and it keeps sneaking in\n uv pip install setuptools==80.8.0\n\n # Restore the artifacts and environment variables\n\n - name: Download shared artifacts\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: shared-artifacts\n path: shared-artifacts\n\n - name: Set environment variables\n run: |\n echo POSTGRES_GIT_REV=$(cat shared-artifacts/postgres_git_rev.txt) >> $GITHUB_ENV\n echo STOLON_GIT_REV=$(cat shared-artifacts/stolon_git_rev.txt) >> $GITHUB_ENV\n echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV\n echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV\n\n # Restore build cache\n\n - name: Restore cached Rust extensions\n uses: actions/cache@v4\n id: rust-cache\n with:\n path: build/rust_extensions\n key: edb-rust-v4-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n\n - name: Restore cached Cython extensions\n uses: actions/cache@v4\n id: ext-cache\n with:\n path: build/extensions\n key: edb-ext-v6-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Restore compiled parsers cache\n uses: actions/cache@v4\n id: parsers-cache\n with:\n path: build/lib\n key: edb-parsers-v3-${{ hashFiles('shared-artifacts/parsers_cache_key.txt') }}\n\n - name: Restore cached PostgreSQL build\n uses: actions/cache@v4\n id: postgres-cache\n with:\n path: build/postgres/install\n key: edb-postgres-v3-${{ env.POSTGRES_GIT_REV }}-${{ hashFiles('shared-artifacts/lib_cache_key.txt') }}\n\n - name: Restore cached Stolon build\n uses: actions/cache@v4\n id: stolon-cache\n with:\n path: build/stolon/bin\n key: edb-stolon-v2-${{ env.STOLON_GIT_REV }}\n\n - name: Restore bootstrap cache\n uses: actions/cache@v4\n id: bootstrap-cache\n with:\n path: build/cache\n key: edb-bootstrap-v2-${{ hashFiles('shared-artifacts/bootstrap_cache_key.txt') }}\n\n - name: Stop if we cannot retrieve the cache\n if: |\n steps.rust-cache.outputs.cache-hit != 'true' ||\n steps.ext-cache.outputs.cache-hit != 'true' ||\n steps.parsers-cache.outputs.cache-hit != 'true' ||\n steps.postgres-cache.outputs.cache-hit != 'true' ||\n steps.stolon-cache.outputs.cache-hit != 'true' ||\n steps.bootstrap-cache.outputs.cache-hit != 'true'\n run: |\n echo ::error::Cannot retrieve build cache.\n exit 1\n\n - name: Validate cached binaries\n run: |\n # Validate Stolon\n ./build/stolon/bin/stolon-sentinel --version || exit 1\n ./build/stolon/bin/stolon-keeper --version || exit 1\n ./build/stolon/bin/stolon-proxy --version || exit 1\n\n # Validate PostgreSQL\n ./build/postgres/install/bin/postgres --version || exit 1\n ./build/postgres/install/bin/pg_config --version || exit 1\n\n - name: Restore cache into the source tree\n run: |\n rsync -av ./build/rust_extensions/edb/ ./edb/\n rsync -av ./build/extensions/edb/ ./edb/\n rsync -av ./build/lib/edb/ ./edb/\n cp build/postgres/install/stamp build/postgres/\n\n - name: Install edgedb-server\n env:\n BUILD_EXT_MODE: skip\n run: |\n # --no-build-isolation because we have explicitly installed all deps\n # and don't want them to be reinstalled in an \"isolated env\".\n pip install --no-build-isolation --no-deps -e .[test,docs]\n\n # Run the test\n\n - name: Test\n env:\n SHARD: ${{ matrix.shard }}\n EDGEDB_TEST_HA: 1\n EDGEDB_TEST_CONSUL_PATH: build/stolon/bin/consul\n EDGEDB_TEST_STOLON_CTL: build/stolon/bin/stolonctl\n EDGEDB_TEST_STOLON_SENTINEL: build/stolon/bin/stolon-sentinel\n EDGEDB_TEST_STOLON_KEEPER: build/stolon/bin/stolon-keeper\n run: |\n edb test -j1 -v -k test_ha_\n\n\n workflow-notifications:\n if: failure() && github.event_name != 'pull_request'\n name: Notify in Slack on failures\n needs:\n - build\n - ha-test\n runs-on: ubuntu-latest\n permissions:\n actions: 'read'\n steps:\n - name: Slack Workflow Notification\n uses: Gamesight/slack-workflow-status@26a36836c887f260477432e4314ec3490a84f309\n with:\n repo_token: ${{secrets.GITHUB_TOKEN}}\n slack_webhook_url: ${{secrets.ACTIONS_SLACK_WEBHOOK_URL}}\n name: 'Workflow notifications'\n icon_emoji: ':hammer:'\n include_jobs: 'on-failure'\n" }, { "path": ".github/workflows/tests.inplace.yml", "content": "name: Tests of in-place upgrades and patching\n\non:\n schedule:\n - cron: \"0 3 * * *\"\n workflow_dispatch:\n inputs: {}\n push:\n branches:\n - \"A-inplace*\"\n\njobs:\n build:\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - uses: actions/checkout@v4\n with:\n fetch-depth: 50\n submodules: true\n\n - name: Set up Python\n uses: actions/setup-python@v5\n id: setup-python\n with:\n python-version: '3.12.2'\n cache: 'pip'\n cache-dependency-path: |\n pyproject.toml\n\n # The below is technically a lie as we are technically not\n # inside a virtual env, but there is really no reason to bother\n # actually creating and activating one as below works just fine.\n - name: Export $VIRTUAL_ENV\n run: |\n venv=\"$(python -c 'import sys; sys.stdout.write(sys.prefix)')\"\n echo \"VIRTUAL_ENV=${venv}\" >> $GITHUB_ENV\n\n - name: Set up uv cache\n uses: actions/cache@v4\n with:\n path: ~/.cache/uv\n key: uv-cache-${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}\n\n - name: Cached requirements.txt\n uses: actions/cache@v4\n id: requirements-cache\n with:\n path: requirements.txt\n key: edb-requirements-${{ hashFiles('pyproject.toml') }}\n\n - name: Compute requirements.txt\n if: steps.requirements-cache.outputs.cache-hit != 'true'\n run: |\n python -m pip install pip-tools\n pip-compile --no-strip-extras --all-build-deps \\\n --extra test,language-server \\\n --output-file requirements.txt pyproject.toml\n\n - name: Install Python dependencies\n run: |\n python -c \"import sys; print(sys.prefix)\"\n python -m pip install uv~=0.1.0 && uv pip install -U -r requirements.txt\n # 80.9.0 breaks our sphinx, and it keeps sneaking in\n uv pip install setuptools==80.8.0\n\n - name: Compute cache keys\n run: |\n mkdir -p shared-artifacts\n if [ \"$(uname)\" = \"Darwin\" ]; then\n find /usr/lib -type f -name 'lib*' -exec stat -f '%N %z' {} + | sort | shasum -a 256 | cut -d ' ' -f1 > shared-artifacts/lib_cache_key.txt\n else\n find /usr/lib -type f -name 'lib*' -printf '%P %s\\n' | sort | sha256sum | cut -d ' ' -f1 > shared-artifacts/lib_cache_key.txt\n fi\n python setup.py -q ci_helper --type rust >shared-artifacts/rust_cache_key.txt\n python setup.py -q ci_helper --type ext >shared-artifacts/ext_cache_key.txt\n python setup.py -q ci_helper --type parsers >shared-artifacts/parsers_cache_key.txt\n python setup.py -q ci_helper --type postgres >shared-artifacts/postgres_git_rev.txt\n python setup.py -q ci_helper --type libpg_query >shared-artifacts/libpg_query_git_rev.txt\n echo 'f8cd94309eaccbfba5dea7835b88c78377608a37' >shared-artifacts/stolon_git_rev.txt\n python setup.py -q ci_helper --type bootstrap >shared-artifacts/bootstrap_cache_key.txt\n echo POSTGRES_GIT_REV=$(cat shared-artifacts/postgres_git_rev.txt) >> $GITHUB_ENV\n echo LIBPG_QUERY_GIT_REV=$(cat shared-artifacts/libpg_query_git_rev.txt) >> $GITHUB_ENV\n echo STOLON_GIT_REV=$(cat shared-artifacts/stolon_git_rev.txt) >> $GITHUB_ENV\n echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV\n echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV\n\n - name: Upload shared artifacts\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: shared-artifacts\n path: shared-artifacts\n retention-days: 1\n\n # Restore binary cache\n\n - name: Handle cached Rust extensions\n uses: actions/cache@v4\n id: rust-cache\n with:\n path: build/rust_extensions\n key: edb-rust-v4-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n restore-keys: |\n edb-rust-v4-\n\n - name: Handle cached Cython extensions\n uses: actions/cache@v4\n id: ext-cache\n with:\n path: build/extensions\n key: edb-ext-v6-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Handle cached PostgreSQL build\n uses: actions/cache@v4\n id: postgres-cache\n with:\n path: build/postgres/install\n key: edb-postgres-v3-${{ env.POSTGRES_GIT_REV }}-${{ hashFiles('shared-artifacts/lib_cache_key.txt') }}\n\n - name: Handle cached Stolon build\n uses: actions/cache@v4\n id: stolon-cache\n with:\n path: build/stolon/bin\n key: edb-stolon-v2-${{ env.STOLON_GIT_REV }}\n\n - name: Handle cached libpg_query build\n uses: actions/cache@v4\n id: libpg-query-cache\n with:\n path: edb/pgsql/parser/libpg_query/libpg_query.a\n key: edb-libpg_query-v1-${{ env.LIBPG_QUERY_GIT_REV }}\n\n # Install system dependencies for building\n\n - name: Install system deps\n if: |\n steps.rust-cache.outputs.cache-hit != 'true' ||\n steps.ext-cache.outputs.cache-hit != 'true' ||\n steps.stolon-cache.outputs.cache-hit != 'true' ||\n steps.postgres-cache.outputs.cache-hit != 'true'\n run: |\n sudo apt-get update\n sudo apt-get install -y uuid-dev libreadline-dev bison flex libprotobuf-c-dev\n\n - name: Install Rust toolchain\n if: steps.rust-cache.outputs.cache-hit != 'true'\n uses: dsherret/rust-toolchain-file@v1\n\n # Build Rust extensions\n\n - name: Handle Rust extensions build cache\n uses: actions/cache@v4\n if: steps.rust-cache.outputs.cache-hit != 'true'\n with:\n path: ${{ env.BUILD_TEMP }}/rust/extensions\n key: edb-rust-build-v1-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n restore-keys: |\n edb-rust-build-v1-\n\n - name: Build Rust extensions\n env:\n CARGO_HOME: ${{ env.BUILD_TEMP }}/rust/extensions/cargo_home\n CACHE_HIT: ${{ steps.rust-cache.outputs.cache-hit }}\n run: |\n if [[ \"$CACHE_HIT\" != \"true\" ]]; then\n rm -rf ${BUILD_LIB}\n mkdir -p build/rust_extensions\n rsync -av ./build/rust_extensions/ ${BUILD_LIB}/\n python setup.py -v build_rust\n rsync -av ${BUILD_LIB}/ build/rust_extensions/\n rm -rf ${BUILD_LIB}\n fi\n rsync -av ./build/rust_extensions/edb/ ./edb/\n\n # Build libpg_query\n\n - name: Build libpg_query\n if: |\n steps.libpg-query-cache.outputs.cache-hit != 'true' &&\n steps.ext-cache.outputs.cache-hit != 'true'\n run: |\n python setup.py build_libpg_query\n\n # Build extensions\n\n - name: Handle Cython extensions build cache\n uses: actions/cache@v4\n if: steps.ext-cache.outputs.cache-hit != 'true'\n with:\n path: ${{ env.BUILD_TEMP }}/edb\n key: edb-ext-build-v4-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Build Cython extensions\n env:\n CACHE_HIT: ${{ steps.ext-cache.outputs.cache-hit }}\n BUILD_EXT_MODE: py-only\n run: |\n if [[ \"$CACHE_HIT\" != \"true\" ]]; then\n rm -rf ${BUILD_LIB}\n mkdir -p ./build/extensions\n rsync -av ./build/extensions/ ${BUILD_LIB}/\n BUILD_EXT_MODE=py-only python setup.py -v build_ext\n rsync -av ${BUILD_LIB}/ ./build/extensions/\n rm -rf ${BUILD_LIB}\n fi\n rsync -av ./build/extensions/edb/ ./edb/\n\n # Build parsers\n\n - name: Handle compiled parsers cache\n uses: actions/cache@v4\n id: parsers-cache\n with:\n path: build/lib\n key: edb-parsers-v3-${{ hashFiles('shared-artifacts/parsers_cache_key.txt') }}\n restore-keys: |\n edb-parsers-v3-\n\n - name: Build parsers\n env:\n CACHE_HIT: ${{ steps.parsers-cache.outputs.cache-hit }}\n run: |\n if [[ \"$CACHE_HIT\" != \"true\" ]]; then\n rm -rf ${BUILD_LIB}\n mkdir -p ./build/lib\n rsync -av ./build/lib/ ${BUILD_LIB}/\n python setup.py -v build_parsers\n rsync -av ${BUILD_LIB}/ ./build/lib/\n rm -rf ${BUILD_LIB}\n fi\n rsync -av ./build/lib/edb/ ./edb/\n\n # Build PostgreSQL\n\n - name: Build PostgreSQL\n env:\n CACHE_HIT: ${{ steps.postgres-cache.outputs.cache-hit }}\n run: |\n if [[ \"$CACHE_HIT\" == \"true\" ]]; then\n cp build/postgres/install/stamp build/postgres/\n else\n python setup.py build_postgres\n cp build/postgres/stamp build/postgres/install/\n fi\n\n # Build Stolon\n\n - name: Set up Go\n if: steps.stolon-cache.outputs.cache-hit != 'true'\n uses: actions/setup-go@v2\n with:\n go-version: 1.16\n\n - uses: actions/checkout@v4\n if: steps.stolon-cache.outputs.cache-hit != 'true'\n with:\n repository: edgedb/stolon\n path: build/stolon\n ref: ${{ env.STOLON_GIT_REV }}\n fetch-depth: 0\n submodules: false\n\n - name: Build Stolon\n if: steps.stolon-cache.outputs.cache-hit != 'true'\n run: |\n mkdir -p build/stolon/bin/\n curl -fsSL https://releases.hashicorp.com/consul/1.10.1/consul_1.10.1_linux_amd64.zip | zcat > build/stolon/bin/consul\n chmod +x build/stolon/bin/consul\n cd build/stolon && make\n\n # Install edgedb-server and populate egg-info\n\n - name: Install edgedb-server\n env:\n BUILD_EXT_MODE: skip\n run: |\n # --no-build-isolation because we have explicitly installed all deps\n # and don't want them to be reinstalled in an \"isolated env\".\n pip install --no-build-isolation --no-deps -e .[test,docs]\n\n # Refresh the bootstrap cache\n\n - name: Handle bootstrap cache\n uses: actions/cache@v4\n id: bootstrap-cache\n with:\n path: build/cache\n key: edb-bootstrap-v2-${{ hashFiles('shared-artifacts/bootstrap_cache_key.txt') }}\n restore-keys: |\n edb-bootstrap-v2-\n\n - name: Bootstrap EdgeDB Server\n if: steps.bootstrap-cache.outputs.cache-hit != 'true'\n run: |\n edb server --bootstrap-only\n\n test-inplace:\n runs-on: ubuntu-latest\n needs: build\n strategy:\n fail-fast: false\n matrix:\n include:\n - flags:\n tests:\n - flags: --rollback-and-test\n tests:\n # Do the reapply test on a smaller selection of tests, since\n # it is slower.\n - flags: --rollback-and-reapply\n tests: -k test_link_on_target_delete -k test_edgeql_select -k test_dump\n\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - uses: actions/checkout@v4\n with:\n fetch-depth: 50\n submodules: true\n\n - name: Set up Python\n uses: actions/setup-python@v5\n id: setup-python\n with:\n python-version: '3.12.2'\n cache: 'pip'\n cache-dependency-path: |\n pyproject.toml\n\n # The below is technically a lie as we are technically not\n # inside a virtual env, but there is really no reason to bother\n # actually creating and activating one as below works just fine.\n - name: Export $VIRTUAL_ENV\n run: |\n venv=\"$(python -c 'import sys; sys.stdout.write(sys.prefix)')\"\n echo \"VIRTUAL_ENV=${venv}\" >> $GITHUB_ENV\n\n - name: Set up uv cache\n uses: actions/cache@v4\n with:\n path: ~/.cache/uv\n key: uv-cache-${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}\n \n - name: Download requirements.txt\n uses: actions/cache@v4\n with:\n path: requirements.txt\n key: edb-requirements-${{ hashFiles('pyproject.toml') }}\n\n - name: Install Python dependencies\n run: |\n python -m pip install uv~=0.1.0 && uv pip install -U -r requirements.txt\n # 80.9.0 breaks our sphinx, and it keeps sneaking in\n uv pip install setuptools==80.8.0\n\n # Restore the artifacts and environment variables\n\n - name: Download shared artifacts\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: shared-artifacts\n path: shared-artifacts\n\n - name: Set environment variables\n run: |\n echo POSTGRES_GIT_REV=$(cat shared-artifacts/postgres_git_rev.txt) >> $GITHUB_ENV\n echo STOLON_GIT_REV=$(cat shared-artifacts/stolon_git_rev.txt) >> $GITHUB_ENV\n echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV\n echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV\n\n # Restore build cache\n\n - name: Restore cached Rust extensions\n uses: actions/cache@v4\n id: rust-cache\n with:\n path: build/rust_extensions\n key: edb-rust-v4-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n\n - name: Restore cached Cython extensions\n uses: actions/cache@v4\n id: ext-cache\n with:\n path: build/extensions\n key: edb-ext-v6-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Restore compiled parsers cache\n uses: actions/cache@v4\n id: parsers-cache\n with:\n path: build/lib\n key: edb-parsers-v3-${{ hashFiles('shared-artifacts/parsers_cache_key.txt') }}\n\n - name: Restore cached PostgreSQL build\n uses: actions/cache@v4\n id: postgres-cache\n with:\n path: build/postgres/install\n key: edb-postgres-v3-${{ env.POSTGRES_GIT_REV }}-${{ hashFiles('shared-artifacts/lib_cache_key.txt') }}\n\n - name: Restore cached Stolon build\n uses: actions/cache@v4\n id: stolon-cache\n with:\n path: build/stolon/bin\n key: edb-stolon-v2-${{ env.STOLON_GIT_REV }}\n\n - name: Restore bootstrap cache\n uses: actions/cache@v4\n id: bootstrap-cache\n with:\n path: build/cache\n key: edb-bootstrap-v2-${{ hashFiles('shared-artifacts/bootstrap_cache_key.txt') }}\n\n - name: Stop if we cannot retrieve the cache\n if: |\n steps.rust-cache.outputs.cache-hit != 'true' ||\n steps.ext-cache.outputs.cache-hit != 'true' ||\n steps.parsers-cache.outputs.cache-hit != 'true' ||\n steps.postgres-cache.outputs.cache-hit != 'true' ||\n steps.stolon-cache.outputs.cache-hit != 'true' ||\n steps.bootstrap-cache.outputs.cache-hit != 'true'\n run: |\n echo ::error::Cannot retrieve build cache.\n exit 1\n\n - name: Validate cached binaries\n run: |\n # Validate Stolon\n ./build/stolon/bin/stolon-sentinel --version || exit 1\n ./build/stolon/bin/stolon-keeper --version || exit 1\n ./build/stolon/bin/stolon-proxy --version || exit 1\n\n # Validate PostgreSQL\n ./build/postgres/install/bin/postgres --version || exit 1\n ./build/postgres/install/bin/pg_config --version || exit 1\n\n - name: Restore cache into the source tree\n run: |\n rsync -av ./build/rust_extensions/edb/ ./edb/\n rsync -av ./build/extensions/edb/ ./edb/\n rsync -av ./build/lib/edb/ ./edb/\n cp build/postgres/install/stamp build/postgres/\n\n - name: Install edgedb-server\n env:\n BUILD_EXT_MODE: skip\n run: |\n # --no-build-isolation because we have explicitly installed all deps\n # and don't want them to be reinstalled in an \"isolated env\".\n pip install --no-build-isolation --no-deps -e .[test,docs]\n\n # Run the test\n # TODO: Would it be better to split this up into multiple jobs?\n - name: Test performing in-place upgrades\n run: |\n ./tests/inplace-testing/test.sh ${{ matrix.flags }} vt ${{ matrix.tests }}\n\n test-patches:\n runs-on: ubuntu-latest\n needs: build\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - uses: actions/checkout@v4\n with:\n fetch-depth: 50\n submodules: true\n\n - name: Set up Python\n uses: actions/setup-python@v5\n id: setup-python\n with:\n python-version: '3.12.2'\n cache: 'pip'\n cache-dependency-path: |\n pyproject.toml\n\n # The below is technically a lie as we are technically not\n # inside a virtual env, but there is really no reason to bother\n # actually creating and activating one as below works just fine.\n - name: Export $VIRTUAL_ENV\n run: |\n venv=\"$(python -c 'import sys; sys.stdout.write(sys.prefix)')\"\n echo \"VIRTUAL_ENV=${venv}\" >> $GITHUB_ENV\n\n - name: Set up uv cache\n uses: actions/cache@v4\n with:\n path: ~/.cache/uv\n key: uv-cache-${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}\n \n - name: Download requirements.txt\n uses: actions/cache@v4\n with:\n path: requirements.txt\n key: edb-requirements-${{ hashFiles('pyproject.toml') }}\n\n - name: Install Python dependencies\n run: |\n python -m pip install uv~=0.1.0 && uv pip install -U -r requirements.txt\n # 80.9.0 breaks our sphinx, and it keeps sneaking in\n uv pip install setuptools==80.8.0\n\n # Restore the artifacts and environment variables\n\n - name: Download shared artifacts\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: shared-artifacts\n path: shared-artifacts\n\n - name: Set environment variables\n run: |\n echo POSTGRES_GIT_REV=$(cat shared-artifacts/postgres_git_rev.txt) >> $GITHUB_ENV\n echo STOLON_GIT_REV=$(cat shared-artifacts/stolon_git_rev.txt) >> $GITHUB_ENV\n echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV\n echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV\n\n # Restore build cache\n\n - name: Restore cached Rust extensions\n uses: actions/cache@v4\n id: rust-cache\n with:\n path: build/rust_extensions\n key: edb-rust-v4-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n\n - name: Restore cached Cython extensions\n uses: actions/cache@v4\n id: ext-cache\n with:\n path: build/extensions\n key: edb-ext-v6-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Restore compiled parsers cache\n uses: actions/cache@v4\n id: parsers-cache\n with:\n path: build/lib\n key: edb-parsers-v3-${{ hashFiles('shared-artifacts/parsers_cache_key.txt') }}\n\n - name: Restore cached PostgreSQL build\n uses: actions/cache@v4\n id: postgres-cache\n with:\n path: build/postgres/install\n key: edb-postgres-v3-${{ env.POSTGRES_GIT_REV }}-${{ hashFiles('shared-artifacts/lib_cache_key.txt') }}\n\n - name: Restore cached Stolon build\n uses: actions/cache@v4\n id: stolon-cache\n with:\n path: build/stolon/bin\n key: edb-stolon-v2-${{ env.STOLON_GIT_REV }}\n\n - name: Restore bootstrap cache\n uses: actions/cache@v4\n id: bootstrap-cache\n with:\n path: build/cache\n key: edb-bootstrap-v2-${{ hashFiles('shared-artifacts/bootstrap_cache_key.txt') }}\n\n - name: Stop if we cannot retrieve the cache\n if: |\n steps.rust-cache.outputs.cache-hit != 'true' ||\n steps.ext-cache.outputs.cache-hit != 'true' ||\n steps.parsers-cache.outputs.cache-hit != 'true' ||\n steps.postgres-cache.outputs.cache-hit != 'true' ||\n steps.stolon-cache.outputs.cache-hit != 'true' ||\n steps.bootstrap-cache.outputs.cache-hit != 'true'\n run: |\n echo ::error::Cannot retrieve build cache.\n exit 1\n\n - name: Validate cached binaries\n run: |\n # Validate Stolon\n ./build/stolon/bin/stolon-sentinel --version || exit 1\n ./build/stolon/bin/stolon-keeper --version || exit 1\n ./build/stolon/bin/stolon-proxy --version || exit 1\n\n # Validate PostgreSQL\n ./build/postgres/install/bin/postgres --version || exit 1\n ./build/postgres/install/bin/pg_config --version || exit 1\n\n - name: Restore cache into the source tree\n run: |\n rsync -av ./build/rust_extensions/edb/ ./edb/\n rsync -av ./build/extensions/edb/ ./edb/\n rsync -av ./build/lib/edb/ ./edb/\n cp build/postgres/install/stamp build/postgres/\n\n - name: Install edgedb-server\n env:\n BUILD_EXT_MODE: skip\n run: |\n # --no-build-isolation because we have explicitly installed all deps\n # and don't want them to be reinstalled in an \"isolated env\".\n pip install --no-build-isolation --no-deps -e .[test,docs]\n\n - name: Test performing in-place upgrades\n run: |\n ./tests/patch-testing/test.sh test-dir -k test_link_on_target_delete -k test_edgeql_select -k test_edgeql_scope -k test_dump\n\n compute-versions:\n runs-on: ubuntu-latest\n outputs:\n matrix: ${{ steps.set-matrix.outputs.matrix }}\n steps:\n - uses: actions/checkout@v4\n - id: set-matrix\n name: Compute versions to run on\n run: python3 .github/scripts/patches/compute-ipu-versions.py\n\n\n test:\n runs-on: ubuntu-latest\n needs: [build, compute-versions]\n strategy:\n fail-fast: false\n matrix: ${{fromJSON(needs.compute-versions.outputs.matrix)}}\n\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - uses: actions/checkout@v4\n with:\n fetch-depth: 50\n submodules: true\n\n - name: Set up Python\n uses: actions/setup-python@v5\n id: setup-python\n with:\n python-version: '3.12.2'\n cache: 'pip'\n cache-dependency-path: |\n pyproject.toml\n\n # The below is technically a lie as we are technically not\n # inside a virtual env, but there is really no reason to bother\n # actually creating and activating one as below works just fine.\n - name: Export $VIRTUAL_ENV\n run: |\n venv=\"$(python -c 'import sys; sys.stdout.write(sys.prefix)')\"\n echo \"VIRTUAL_ENV=${venv}\" >> $GITHUB_ENV\n\n - name: Set up uv cache\n uses: actions/cache@v4\n with:\n path: ~/.cache/uv\n key: uv-cache-${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}\n \n - name: Download requirements.txt\n uses: actions/cache@v4\n with:\n path: requirements.txt\n key: edb-requirements-${{ hashFiles('pyproject.toml') }}\n\n - name: Install Python dependencies\n run: |\n python -m pip install uv~=0.1.0 && uv pip install -U -r requirements.txt\n # 80.9.0 breaks our sphinx, and it keeps sneaking in\n uv pip install setuptools==80.8.0\n\n # Restore the artifacts and environment variables\n\n - name: Download shared artifacts\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: shared-artifacts\n path: shared-artifacts\n\n - name: Set environment variables\n run: |\n echo POSTGRES_GIT_REV=$(cat shared-artifacts/postgres_git_rev.txt) >> $GITHUB_ENV\n echo STOLON_GIT_REV=$(cat shared-artifacts/stolon_git_rev.txt) >> $GITHUB_ENV\n echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV\n echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV\n\n # Restore build cache\n\n - name: Restore cached Rust extensions\n uses: actions/cache@v4\n id: rust-cache\n with:\n path: build/rust_extensions\n key: edb-rust-v4-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n\n - name: Restore cached Cython extensions\n uses: actions/cache@v4\n id: ext-cache\n with:\n path: build/extensions\n key: edb-ext-v6-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Restore compiled parsers cache\n uses: actions/cache@v4\n id: parsers-cache\n with:\n path: build/lib\n key: edb-parsers-v3-${{ hashFiles('shared-artifacts/parsers_cache_key.txt') }}\n\n - name: Restore cached PostgreSQL build\n uses: actions/cache@v4\n id: postgres-cache\n with:\n path: build/postgres/install\n key: edb-postgres-v3-${{ env.POSTGRES_GIT_REV }}-${{ hashFiles('shared-artifacts/lib_cache_key.txt') }}\n\n - name: Restore cached Stolon build\n uses: actions/cache@v4\n id: stolon-cache\n with:\n path: build/stolon/bin\n key: edb-stolon-v2-${{ env.STOLON_GIT_REV }}\n\n - name: Restore bootstrap cache\n uses: actions/cache@v4\n id: bootstrap-cache\n with:\n path: build/cache\n key: edb-bootstrap-v2-${{ hashFiles('shared-artifacts/bootstrap_cache_key.txt') }}\n\n - name: Stop if we cannot retrieve the cache\n if: |\n steps.rust-cache.outputs.cache-hit != 'true' ||\n steps.ext-cache.outputs.cache-hit != 'true' ||\n steps.parsers-cache.outputs.cache-hit != 'true' ||\n steps.postgres-cache.outputs.cache-hit != 'true' ||\n steps.stolon-cache.outputs.cache-hit != 'true' ||\n steps.bootstrap-cache.outputs.cache-hit != 'true'\n run: |\n echo ::error::Cannot retrieve build cache.\n exit 1\n\n - name: Validate cached binaries\n run: |\n # Validate Stolon\n ./build/stolon/bin/stolon-sentinel --version || exit 1\n ./build/stolon/bin/stolon-keeper --version || exit 1\n ./build/stolon/bin/stolon-proxy --version || exit 1\n\n # Validate PostgreSQL\n ./build/postgres/install/bin/postgres --version || exit 1\n ./build/postgres/install/bin/pg_config --version || exit 1\n\n - name: Restore cache into the source tree\n run: |\n rsync -av ./build/rust_extensions/edb/ ./edb/\n rsync -av ./build/extensions/edb/ ./edb/\n rsync -av ./build/lib/edb/ ./edb/\n cp build/postgres/install/stamp build/postgres/\n\n - name: Install edgedb-server\n env:\n BUILD_EXT_MODE: skip\n run: |\n # --no-build-isolation because we have explicitly installed all deps\n # and don't want them to be reinstalled in an \"isolated env\".\n pip install --no-build-isolation --no-deps -e .[test,docs]\n\n # Run the test\n\n - name: Download an earlier database version\n run: |\n wget -q \"${{ matrix.edgedb-url }}\"\n tar xzf ${{ matrix.edgedb-basename }}-${{ matrix.edgedb-version }}.tar.gz\n\n - name: Make sure a CLI named \"edgedb\" exists (sigh)\n run: |\n ln -s gel $(dirname $(which gel))/edgedb\n\n - name: Test inplace upgrades from previous major version\n run: |\n ./tests/inplace-testing/test-old.sh vt ${{ matrix.edgedb-basename }}-${{ matrix.edgedb-version }}\n\n\n workflow-notifications:\n if: failure() && github.event_name != 'pull_request'\n name: Notify in Slack on failures\n needs:\n - build\n - test-inplace\n - test-patches\n runs-on: ubuntu-latest\n permissions:\n actions: 'read'\n steps:\n - name: Slack Workflow Notification\n uses: Gamesight/slack-workflow-status@26a36836c887f260477432e4314ec3490a84f309\n with:\n repo_token: ${{secrets.GITHUB_TOKEN}}\n slack_webhook_url: ${{secrets.ACTIONS_SLACK_WEBHOOK_URL}}\n name: 'Workflow notifications'\n icon_emoji: ':hammer:'\n include_jobs: 'on-failure'\n" }, { "path": ".github/workflows/tests.inplace7x.yml", "content": "name: Tests of in-place upgrades to 7.x\n\non:\n schedule:\n - cron: \"0 3 * * *\"\n workflow_dispatch:\n inputs: {}\n push:\n branches:\n - \"A-inplace*\"\n\njobs:\n build:\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n ref: release/7.x\n\n - uses: actions/checkout@v4\n with:\n fetch-depth: 50\n submodules: true\n ref: release/7.x\n\n - name: Set up Python\n uses: actions/setup-python@v5\n id: setup-python\n with:\n python-version: '3.12.2'\n cache: 'pip'\n cache-dependency-path: |\n pyproject.toml\n\n # The below is technically a lie as we are technically not\n # inside a virtual env, but there is really no reason to bother\n # actually creating and activating one as below works just fine.\n - name: Export $VIRTUAL_ENV\n run: |\n venv=\"$(python -c 'import sys; sys.stdout.write(sys.prefix)')\"\n echo \"VIRTUAL_ENV=${venv}\" >> $GITHUB_ENV\n\n - name: Set up uv cache\n uses: actions/cache@v4\n with:\n path: ~/.cache/uv\n key: uv-cache-${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}\n\n - name: Cached requirements.txt\n uses: actions/cache@v4\n id: requirements-cache\n with:\n path: requirements.txt\n key: edb-requirements-${{ hashFiles('pyproject.toml') }}\n\n - name: Compute requirements.txt\n if: steps.requirements-cache.outputs.cache-hit != 'true'\n run: |\n python -m pip install pip-tools\n pip-compile --no-strip-extras --all-build-deps \\\n --extra test,language-server \\\n --output-file requirements.txt pyproject.toml\n\n - name: Install Python dependencies\n run: |\n python -c \"import sys; print(sys.prefix)\"\n python -m pip install uv~=0.1.0 && uv pip install -U -r requirements.txt\n # 80.9.0 breaks our sphinx, and it keeps sneaking in\n uv pip install setuptools==80.8.0\n\n - name: Compute cache keys\n run: |\n mkdir -p shared-artifacts\n if [ \"$(uname)\" = \"Darwin\" ]; then\n find /usr/lib -type f -name 'lib*' -exec stat -f '%N %z' {} + | sort | shasum -a 256 | cut -d ' ' -f1 > shared-artifacts/lib_cache_key.txt\n else\n find /usr/lib -type f -name 'lib*' -printf '%P %s\\n' | sort | sha256sum | cut -d ' ' -f1 > shared-artifacts/lib_cache_key.txt\n fi\n python setup.py -q ci_helper --type rust >shared-artifacts/rust_cache_key.txt\n python setup.py -q ci_helper --type ext >shared-artifacts/ext_cache_key.txt\n python setup.py -q ci_helper --type parsers >shared-artifacts/parsers_cache_key.txt\n python setup.py -q ci_helper --type postgres >shared-artifacts/postgres_git_rev.txt\n python setup.py -q ci_helper --type libpg_query >shared-artifacts/libpg_query_git_rev.txt\n echo 'f8cd94309eaccbfba5dea7835b88c78377608a37' >shared-artifacts/stolon_git_rev.txt\n python setup.py -q ci_helper --type bootstrap >shared-artifacts/bootstrap_cache_key.txt\n echo POSTGRES_GIT_REV=$(cat shared-artifacts/postgres_git_rev.txt) >> $GITHUB_ENV\n echo LIBPG_QUERY_GIT_REV=$(cat shared-artifacts/libpg_query_git_rev.txt) >> $GITHUB_ENV\n echo STOLON_GIT_REV=$(cat shared-artifacts/stolon_git_rev.txt) >> $GITHUB_ENV\n echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV\n echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV\n\n - name: Upload shared artifacts\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: shared-artifacts\n path: shared-artifacts\n retention-days: 1\n\n # Restore binary cache\n\n - name: Handle cached Rust extensions\n uses: actions/cache@v4\n id: rust-cache\n with:\n path: build/rust_extensions\n key: edb-rust-v4-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n restore-keys: |\n edb-rust-v4-\n\n - name: Handle cached Cython extensions\n uses: actions/cache@v4\n id: ext-cache\n with:\n path: build/extensions\n key: edb-ext-v6-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Handle cached PostgreSQL build\n uses: actions/cache@v4\n id: postgres-cache\n with:\n path: build/postgres/install\n key: edb-postgres-v3-${{ env.POSTGRES_GIT_REV }}-${{ hashFiles('shared-artifacts/lib_cache_key.txt') }}\n\n - name: Handle cached Stolon build\n uses: actions/cache@v4\n id: stolon-cache\n with:\n path: build/stolon/bin\n key: edb-stolon-v2-${{ env.STOLON_GIT_REV }}\n\n - name: Handle cached libpg_query build\n uses: actions/cache@v4\n id: libpg-query-cache\n with:\n path: edb/pgsql/parser/libpg_query/libpg_query.a\n key: edb-libpg_query-v1-${{ env.LIBPG_QUERY_GIT_REV }}\n\n # Install system dependencies for building\n\n - name: Install system deps\n if: |\n steps.rust-cache.outputs.cache-hit != 'true' ||\n steps.ext-cache.outputs.cache-hit != 'true' ||\n steps.stolon-cache.outputs.cache-hit != 'true' ||\n steps.postgres-cache.outputs.cache-hit != 'true'\n run: |\n sudo apt-get update\n sudo apt-get install -y uuid-dev libreadline-dev bison flex libprotobuf-c-dev\n\n - name: Install Rust toolchain\n if: steps.rust-cache.outputs.cache-hit != 'true'\n uses: dsherret/rust-toolchain-file@v1\n\n # Build Rust extensions\n\n - name: Handle Rust extensions build cache\n uses: actions/cache@v4\n if: steps.rust-cache.outputs.cache-hit != 'true'\n with:\n path: ${{ env.BUILD_TEMP }}/rust/extensions\n key: edb-rust-build-v1-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n restore-keys: |\n edb-rust-build-v1-\n\n - name: Build Rust extensions\n env:\n CARGO_HOME: ${{ env.BUILD_TEMP }}/rust/extensions/cargo_home\n CACHE_HIT: ${{ steps.rust-cache.outputs.cache-hit }}\n run: |\n if [[ \"$CACHE_HIT\" != \"true\" ]]; then\n rm -rf ${BUILD_LIB}\n mkdir -p build/rust_extensions\n rsync -av ./build/rust_extensions/ ${BUILD_LIB}/\n python setup.py -v build_rust\n rsync -av ${BUILD_LIB}/ build/rust_extensions/\n rm -rf ${BUILD_LIB}\n fi\n rsync -av ./build/rust_extensions/edb/ ./edb/\n\n # Build libpg_query\n\n - name: Build libpg_query\n if: |\n steps.libpg-query-cache.outputs.cache-hit != 'true' &&\n steps.ext-cache.outputs.cache-hit != 'true'\n run: |\n python setup.py build_libpg_query\n\n # Build extensions\n\n - name: Handle Cython extensions build cache\n uses: actions/cache@v4\n if: steps.ext-cache.outputs.cache-hit != 'true'\n with:\n path: ${{ env.BUILD_TEMP }}/edb\n key: edb-ext-build-v4-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Build Cython extensions\n env:\n CACHE_HIT: ${{ steps.ext-cache.outputs.cache-hit }}\n BUILD_EXT_MODE: py-only\n run: |\n if [[ \"$CACHE_HIT\" != \"true\" ]]; then\n rm -rf ${BUILD_LIB}\n mkdir -p ./build/extensions\n rsync -av ./build/extensions/ ${BUILD_LIB}/\n BUILD_EXT_MODE=py-only python setup.py -v build_ext\n rsync -av ${BUILD_LIB}/ ./build/extensions/\n rm -rf ${BUILD_LIB}\n fi\n rsync -av ./build/extensions/edb/ ./edb/\n\n # Build parsers\n\n - name: Handle compiled parsers cache\n uses: actions/cache@v4\n id: parsers-cache\n with:\n path: build/lib\n key: edb-parsers-v3-${{ hashFiles('shared-artifacts/parsers_cache_key.txt') }}\n restore-keys: |\n edb-parsers-v3-\n\n - name: Build parsers\n env:\n CACHE_HIT: ${{ steps.parsers-cache.outputs.cache-hit }}\n run: |\n if [[ \"$CACHE_HIT\" != \"true\" ]]; then\n rm -rf ${BUILD_LIB}\n mkdir -p ./build/lib\n rsync -av ./build/lib/ ${BUILD_LIB}/\n python setup.py -v build_parsers\n rsync -av ${BUILD_LIB}/ ./build/lib/\n rm -rf ${BUILD_LIB}\n fi\n rsync -av ./build/lib/edb/ ./edb/\n\n # Build PostgreSQL\n\n - name: Build PostgreSQL\n env:\n CACHE_HIT: ${{ steps.postgres-cache.outputs.cache-hit }}\n run: |\n if [[ \"$CACHE_HIT\" == \"true\" ]]; then\n cp build/postgres/install/stamp build/postgres/\n else\n python setup.py build_postgres\n cp build/postgres/stamp build/postgres/install/\n fi\n\n # Build Stolon\n\n - name: Set up Go\n if: steps.stolon-cache.outputs.cache-hit != 'true'\n uses: actions/setup-go@v2\n with:\n go-version: 1.16\n\n - uses: actions/checkout@v4\n if: steps.stolon-cache.outputs.cache-hit != 'true'\n with:\n repository: edgedb/stolon\n path: build/stolon\n ref: ${{ env.STOLON_GIT_REV }}\n fetch-depth: 0\n submodules: false\n\n - name: Build Stolon\n if: steps.stolon-cache.outputs.cache-hit != 'true'\n run: |\n mkdir -p build/stolon/bin/\n curl -fsSL https://releases.hashicorp.com/consul/1.10.1/consul_1.10.1_linux_amd64.zip | zcat > build/stolon/bin/consul\n chmod +x build/stolon/bin/consul\n cd build/stolon && make\n\n # Install edgedb-server and populate egg-info\n\n - name: Install edgedb-server\n env:\n BUILD_EXT_MODE: skip\n run: |\n # --no-build-isolation because we have explicitly installed all deps\n # and don't want them to be reinstalled in an \"isolated env\".\n pip install --no-build-isolation --no-deps -e .[test,docs]\n\n # Refresh the bootstrap cache\n\n - name: Handle bootstrap cache\n uses: actions/cache@v4\n id: bootstrap-cache\n with:\n path: build/cache\n key: edb-bootstrap-v2-${{ hashFiles('shared-artifacts/bootstrap_cache_key.txt') }}\n restore-keys: |\n edb-bootstrap-v2-\n\n - name: Bootstrap EdgeDB Server\n if: steps.bootstrap-cache.outputs.cache-hit != 'true'\n run: |\n edb server --bootstrap-only\n\n test-inplace:\n runs-on: ubuntu-latest\n needs: build\n strategy:\n fail-fast: false\n matrix:\n include:\n - flags:\n tests:\n - flags: --rollback-and-test\n tests:\n # Do the reapply test on a smaller selection of tests, since\n # it is slower.\n - flags: --rollback-and-reapply\n tests: -k test_link_on_target_delete -k test_edgeql_select -k test_dump\n\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n ref: release/7.x\n\n - uses: actions/checkout@v4\n with:\n fetch-depth: 50\n submodules: true\n ref: release/7.x\n\n - name: Set up Python\n uses: actions/setup-python@v5\n id: setup-python\n with:\n python-version: '3.12.2'\n cache: 'pip'\n cache-dependency-path: |\n pyproject.toml\n\n # The below is technically a lie as we are technically not\n # inside a virtual env, but there is really no reason to bother\n # actually creating and activating one as below works just fine.\n - name: Export $VIRTUAL_ENV\n run: |\n venv=\"$(python -c 'import sys; sys.stdout.write(sys.prefix)')\"\n echo \"VIRTUAL_ENV=${venv}\" >> $GITHUB_ENV\n\n - name: Set up uv cache\n uses: actions/cache@v4\n with:\n path: ~/.cache/uv\n key: uv-cache-${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}\n \n - name: Download requirements.txt\n uses: actions/cache@v4\n with:\n path: requirements.txt\n key: edb-requirements-${{ hashFiles('pyproject.toml') }}\n\n - name: Install Python dependencies\n run: |\n python -m pip install uv~=0.1.0 && uv pip install -U -r requirements.txt\n # 80.9.0 breaks our sphinx, and it keeps sneaking in\n uv pip install setuptools==80.8.0\n\n # Restore the artifacts and environment variables\n\n - name: Download shared artifacts\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: shared-artifacts\n path: shared-artifacts\n\n - name: Set environment variables\n run: |\n echo POSTGRES_GIT_REV=$(cat shared-artifacts/postgres_git_rev.txt) >> $GITHUB_ENV\n echo STOLON_GIT_REV=$(cat shared-artifacts/stolon_git_rev.txt) >> $GITHUB_ENV\n echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV\n echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV\n\n # Restore build cache\n\n - name: Restore cached Rust extensions\n uses: actions/cache@v4\n id: rust-cache\n with:\n path: build/rust_extensions\n key: edb-rust-v4-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n\n - name: Restore cached Cython extensions\n uses: actions/cache@v4\n id: ext-cache\n with:\n path: build/extensions\n key: edb-ext-v6-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Restore compiled parsers cache\n uses: actions/cache@v4\n id: parsers-cache\n with:\n path: build/lib\n key: edb-parsers-v3-${{ hashFiles('shared-artifacts/parsers_cache_key.txt') }}\n\n - name: Restore cached PostgreSQL build\n uses: actions/cache@v4\n id: postgres-cache\n with:\n path: build/postgres/install\n key: edb-postgres-v3-${{ env.POSTGRES_GIT_REV }}-${{ hashFiles('shared-artifacts/lib_cache_key.txt') }}\n\n - name: Restore cached Stolon build\n uses: actions/cache@v4\n id: stolon-cache\n with:\n path: build/stolon/bin\n key: edb-stolon-v2-${{ env.STOLON_GIT_REV }}\n\n - name: Restore bootstrap cache\n uses: actions/cache@v4\n id: bootstrap-cache\n with:\n path: build/cache\n key: edb-bootstrap-v2-${{ hashFiles('shared-artifacts/bootstrap_cache_key.txt') }}\n\n - name: Stop if we cannot retrieve the cache\n if: |\n steps.rust-cache.outputs.cache-hit != 'true' ||\n steps.ext-cache.outputs.cache-hit != 'true' ||\n steps.parsers-cache.outputs.cache-hit != 'true' ||\n steps.postgres-cache.outputs.cache-hit != 'true' ||\n steps.stolon-cache.outputs.cache-hit != 'true' ||\n steps.bootstrap-cache.outputs.cache-hit != 'true'\n run: |\n echo ::error::Cannot retrieve build cache.\n exit 1\n\n - name: Validate cached binaries\n run: |\n # Validate Stolon\n ./build/stolon/bin/stolon-sentinel --version || exit 1\n ./build/stolon/bin/stolon-keeper --version || exit 1\n ./build/stolon/bin/stolon-proxy --version || exit 1\n\n # Validate PostgreSQL\n ./build/postgres/install/bin/postgres --version || exit 1\n ./build/postgres/install/bin/pg_config --version || exit 1\n\n - name: Restore cache into the source tree\n run: |\n rsync -av ./build/rust_extensions/edb/ ./edb/\n rsync -av ./build/extensions/edb/ ./edb/\n rsync -av ./build/lib/edb/ ./edb/\n cp build/postgres/install/stamp build/postgres/\n\n - name: Install edgedb-server\n env:\n BUILD_EXT_MODE: skip\n run: |\n # --no-build-isolation because we have explicitly installed all deps\n # and don't want them to be reinstalled in an \"isolated env\".\n pip install --no-build-isolation --no-deps -e .[test,docs]\n\n # Run the test\n # TODO: Would it be better to split this up into multiple jobs?\n - name: Test performing in-place upgrades\n run: |\n ./tests/inplace-testing/test.sh ${{ matrix.flags }} vt ${{ matrix.tests }}\n\n compute-versions:\n runs-on: ubuntu-latest\n outputs:\n matrix: ${{ steps.set-matrix.outputs.matrix }}\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n ref: release/7.x\n - id: set-matrix\n name: Compute versions to run on\n run: python3 .github/scripts/patches/compute-ipu-versions.py\n\n\n test:\n runs-on: ubuntu-latest\n needs: [build, compute-versions]\n strategy:\n fail-fast: false\n matrix: ${{fromJSON(needs.compute-versions.outputs.matrix)}}\n\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n ref: release/7.x\n\n - uses: actions/checkout@v4\n with:\n fetch-depth: 50\n submodules: true\n ref: release/7.x\n\n - name: Set up Python\n uses: actions/setup-python@v5\n id: setup-python\n with:\n python-version: '3.12.2'\n cache: 'pip'\n cache-dependency-path: |\n pyproject.toml\n\n # The below is technically a lie as we are technically not\n # inside a virtual env, but there is really no reason to bother\n # actually creating and activating one as below works just fine.\n - name: Export $VIRTUAL_ENV\n run: |\n venv=\"$(python -c 'import sys; sys.stdout.write(sys.prefix)')\"\n echo \"VIRTUAL_ENV=${venv}\" >> $GITHUB_ENV\n\n - name: Set up uv cache\n uses: actions/cache@v4\n with:\n path: ~/.cache/uv\n key: uv-cache-${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}\n \n - name: Download requirements.txt\n uses: actions/cache@v4\n with:\n path: requirements.txt\n key: edb-requirements-${{ hashFiles('pyproject.toml') }}\n\n - name: Install Python dependencies\n run: |\n python -m pip install uv~=0.1.0 && uv pip install -U -r requirements.txt\n # 80.9.0 breaks our sphinx, and it keeps sneaking in\n uv pip install setuptools==80.8.0\n\n # Restore the artifacts and environment variables\n\n - name: Download shared artifacts\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: shared-artifacts\n path: shared-artifacts\n\n - name: Set environment variables\n run: |\n echo POSTGRES_GIT_REV=$(cat shared-artifacts/postgres_git_rev.txt) >> $GITHUB_ENV\n echo STOLON_GIT_REV=$(cat shared-artifacts/stolon_git_rev.txt) >> $GITHUB_ENV\n echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV\n echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV\n\n # Restore build cache\n\n - name: Restore cached Rust extensions\n uses: actions/cache@v4\n id: rust-cache\n with:\n path: build/rust_extensions\n key: edb-rust-v4-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n\n - name: Restore cached Cython extensions\n uses: actions/cache@v4\n id: ext-cache\n with:\n path: build/extensions\n key: edb-ext-v6-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Restore compiled parsers cache\n uses: actions/cache@v4\n id: parsers-cache\n with:\n path: build/lib\n key: edb-parsers-v3-${{ hashFiles('shared-artifacts/parsers_cache_key.txt') }}\n\n - name: Restore cached PostgreSQL build\n uses: actions/cache@v4\n id: postgres-cache\n with:\n path: build/postgres/install\n key: edb-postgres-v3-${{ env.POSTGRES_GIT_REV }}-${{ hashFiles('shared-artifacts/lib_cache_key.txt') }}\n\n - name: Restore cached Stolon build\n uses: actions/cache@v4\n id: stolon-cache\n with:\n path: build/stolon/bin\n key: edb-stolon-v2-${{ env.STOLON_GIT_REV }}\n\n - name: Restore bootstrap cache\n uses: actions/cache@v4\n id: bootstrap-cache\n with:\n path: build/cache\n key: edb-bootstrap-v2-${{ hashFiles('shared-artifacts/bootstrap_cache_key.txt') }}\n\n - name: Stop if we cannot retrieve the cache\n if: |\n steps.rust-cache.outputs.cache-hit != 'true' ||\n steps.ext-cache.outputs.cache-hit != 'true' ||\n steps.parsers-cache.outputs.cache-hit != 'true' ||\n steps.postgres-cache.outputs.cache-hit != 'true' ||\n steps.stolon-cache.outputs.cache-hit != 'true' ||\n steps.bootstrap-cache.outputs.cache-hit != 'true'\n run: |\n echo ::error::Cannot retrieve build cache.\n exit 1\n\n - name: Validate cached binaries\n run: |\n # Validate Stolon\n ./build/stolon/bin/stolon-sentinel --version || exit 1\n ./build/stolon/bin/stolon-keeper --version || exit 1\n ./build/stolon/bin/stolon-proxy --version || exit 1\n\n # Validate PostgreSQL\n ./build/postgres/install/bin/postgres --version || exit 1\n ./build/postgres/install/bin/pg_config --version || exit 1\n\n - name: Restore cache into the source tree\n run: |\n rsync -av ./build/rust_extensions/edb/ ./edb/\n rsync -av ./build/extensions/edb/ ./edb/\n rsync -av ./build/lib/edb/ ./edb/\n cp build/postgres/install/stamp build/postgres/\n\n - name: Install edgedb-server\n env:\n BUILD_EXT_MODE: skip\n run: |\n # --no-build-isolation because we have explicitly installed all deps\n # and don't want them to be reinstalled in an \"isolated env\".\n pip install --no-build-isolation --no-deps -e .[test,docs]\n\n # Run the test\n\n - name: Download an earlier database version\n run: |\n wget -q \"${{ matrix.edgedb-url }}\"\n tar xzf ${{ matrix.edgedb-basename }}-${{ matrix.edgedb-version }}.tar.gz\n\n - name: Make sure a CLI named \"edgedb\" exists (sigh)\n run: |\n ln -s gel $(dirname $(which gel))/edgedb\n\n - name: Test inplace upgrades from previous major version\n run: |\n ./tests/inplace-testing/test-old.sh vt ${{ matrix.edgedb-basename }}-${{ matrix.edgedb-version }}\n\n\n workflow-notifications:\n if: failure() && github.event_name != 'pull_request'\n name: Notify in Slack on failures\n needs:\n - build\n - test-inplace\n runs-on: ubuntu-latest\n permissions:\n actions: 'read'\n steps:\n - name: Slack Workflow Notification\n uses: Gamesight/slack-workflow-status@26a36836c887f260477432e4314ec3490a84f309\n with:\n repo_token: ${{secrets.GITHUB_TOKEN}}\n slack_webhook_url: ${{secrets.ACTIONS_SLACK_WEBHOOK_URL}}\n name: 'Workflow notifications'\n icon_emoji: ':hammer:'\n include_jobs: 'on-failure'\n" }, { "path": ".github/workflows/tests.managed-pg.yml", "content": "name: Tests on Managed PostgreSQL\n\non:\n schedule:\n - cron: \"0 3 * * 6\"\n workflow_dispatch:\n inputs: {}\n push:\n branches:\n - cloud-test\n\njobs:\n build:\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - uses: actions/checkout@v4\n with:\n fetch-depth: 50\n submodules: true\n\n - name: Set up Python\n uses: actions/setup-python@v5\n id: setup-python\n with:\n python-version: '3.12.2'\n cache: 'pip'\n cache-dependency-path: |\n pyproject.toml\n\n # The below is technically a lie as we are technically not\n # inside a virtual env, but there is really no reason to bother\n # actually creating and activating one as below works just fine.\n - name: Export $VIRTUAL_ENV\n run: |\n venv=\"$(python -c 'import sys; sys.stdout.write(sys.prefix)')\"\n echo \"VIRTUAL_ENV=${venv}\" >> $GITHUB_ENV\n\n - name: Set up uv cache\n uses: actions/cache@v4\n with:\n path: ~/.cache/uv\n key: uv-cache-${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}\n\n - name: Cached requirements.txt\n uses: actions/cache@v4\n id: requirements-cache\n with:\n path: requirements.txt\n key: edb-requirements-${{ hashFiles('pyproject.toml') }}\n\n - name: Compute requirements.txt\n if: steps.requirements-cache.outputs.cache-hit != 'true'\n run: |\n python -m pip install pip-tools\n pip-compile --no-strip-extras --all-build-deps \\\n --extra test,language-server \\\n --output-file requirements.txt pyproject.toml\n\n - name: Install Python dependencies\n run: |\n python -c \"import sys; print(sys.prefix)\"\n python -m pip install uv~=0.1.0 && uv pip install -U -r requirements.txt\n # 80.9.0 breaks our sphinx, and it keeps sneaking in\n uv pip install setuptools==80.8.0\n\n - name: Compute cache keys\n run: |\n mkdir -p shared-artifacts\n if [ \"$(uname)\" = \"Darwin\" ]; then\n find /usr/lib -type f -name 'lib*' -exec stat -f '%N %z' {} + | sort | shasum -a 256 | cut -d ' ' -f1 > shared-artifacts/lib_cache_key.txt\n else\n find /usr/lib -type f -name 'lib*' -printf '%P %s\\n' | sort | sha256sum | cut -d ' ' -f1 > shared-artifacts/lib_cache_key.txt\n fi\n python setup.py -q ci_helper --type rust >shared-artifacts/rust_cache_key.txt\n python setup.py -q ci_helper --type ext >shared-artifacts/ext_cache_key.txt\n python setup.py -q ci_helper --type parsers >shared-artifacts/parsers_cache_key.txt\n python setup.py -q ci_helper --type postgres >shared-artifacts/postgres_git_rev.txt\n python setup.py -q ci_helper --type libpg_query >shared-artifacts/libpg_query_git_rev.txt\n echo 'f8cd94309eaccbfba5dea7835b88c78377608a37' >shared-artifacts/stolon_git_rev.txt\n python setup.py -q ci_helper --type bootstrap >shared-artifacts/bootstrap_cache_key.txt\n echo POSTGRES_GIT_REV=$(cat shared-artifacts/postgres_git_rev.txt) >> $GITHUB_ENV\n echo LIBPG_QUERY_GIT_REV=$(cat shared-artifacts/libpg_query_git_rev.txt) >> $GITHUB_ENV\n echo STOLON_GIT_REV=$(cat shared-artifacts/stolon_git_rev.txt) >> $GITHUB_ENV\n echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV\n echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV\n\n - name: Upload shared artifacts\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: shared-artifacts\n path: shared-artifacts\n retention-days: 1\n\n # Restore binary cache\n\n - name: Handle cached Rust extensions\n uses: actions/cache@v4\n id: rust-cache\n with:\n path: build/rust_extensions\n key: edb-rust-v4-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n restore-keys: |\n edb-rust-v4-\n\n - name: Handle cached Cython extensions\n uses: actions/cache@v4\n id: ext-cache\n with:\n path: build/extensions\n key: edb-ext-v6-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Handle cached PostgreSQL build\n uses: actions/cache@v4\n id: postgres-cache\n with:\n path: build/postgres/install\n key: edb-postgres-v3-${{ env.POSTGRES_GIT_REV }}-${{ hashFiles('shared-artifacts/lib_cache_key.txt') }}\n\n - name: Handle cached Stolon build\n uses: actions/cache@v4\n id: stolon-cache\n with:\n path: build/stolon/bin\n key: edb-stolon-v2-${{ env.STOLON_GIT_REV }}\n\n - name: Handle cached libpg_query build\n uses: actions/cache@v4\n id: libpg-query-cache\n with:\n path: edb/pgsql/parser/libpg_query/libpg_query.a\n key: edb-libpg_query-v1-${{ env.LIBPG_QUERY_GIT_REV }}\n\n # Install system dependencies for building\n\n - name: Install system deps\n if: |\n steps.rust-cache.outputs.cache-hit != 'true' ||\n steps.ext-cache.outputs.cache-hit != 'true' ||\n steps.stolon-cache.outputs.cache-hit != 'true' ||\n steps.postgres-cache.outputs.cache-hit != 'true'\n run: |\n sudo apt-get update\n sudo apt-get install -y uuid-dev libreadline-dev bison flex libprotobuf-c-dev\n\n - name: Install Rust toolchain\n if: steps.rust-cache.outputs.cache-hit != 'true'\n uses: dsherret/rust-toolchain-file@v1\n\n # Build Rust extensions\n\n - name: Handle Rust extensions build cache\n uses: actions/cache@v4\n if: steps.rust-cache.outputs.cache-hit != 'true'\n with:\n path: ${{ env.BUILD_TEMP }}/rust/extensions\n key: edb-rust-build-v1-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n restore-keys: |\n edb-rust-build-v1-\n\n - name: Build Rust extensions\n env:\n CARGO_HOME: ${{ env.BUILD_TEMP }}/rust/extensions/cargo_home\n CACHE_HIT: ${{ steps.rust-cache.outputs.cache-hit }}\n run: |\n if [[ \"$CACHE_HIT\" != \"true\" ]]; then\n rm -rf ${BUILD_LIB}\n mkdir -p build/rust_extensions\n rsync -av ./build/rust_extensions/ ${BUILD_LIB}/\n python setup.py -v build_rust\n rsync -av ${BUILD_LIB}/ build/rust_extensions/\n rm -rf ${BUILD_LIB}\n fi\n rsync -av ./build/rust_extensions/edb/ ./edb/\n\n # Build libpg_query\n\n - name: Build libpg_query\n if: |\n steps.libpg-query-cache.outputs.cache-hit != 'true' &&\n steps.ext-cache.outputs.cache-hit != 'true'\n run: |\n python setup.py build_libpg_query\n\n # Build extensions\n\n - name: Handle Cython extensions build cache\n uses: actions/cache@v4\n if: steps.ext-cache.outputs.cache-hit != 'true'\n with:\n path: ${{ env.BUILD_TEMP }}/edb\n key: edb-ext-build-v4-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Build Cython extensions\n env:\n CACHE_HIT: ${{ steps.ext-cache.outputs.cache-hit }}\n BUILD_EXT_MODE: py-only\n run: |\n if [[ \"$CACHE_HIT\" != \"true\" ]]; then\n rm -rf ${BUILD_LIB}\n mkdir -p ./build/extensions\n rsync -av ./build/extensions/ ${BUILD_LIB}/\n BUILD_EXT_MODE=py-only python setup.py -v build_ext\n rsync -av ${BUILD_LIB}/ ./build/extensions/\n rm -rf ${BUILD_LIB}\n fi\n rsync -av ./build/extensions/edb/ ./edb/\n\n # Build parsers\n\n - name: Handle compiled parsers cache\n uses: actions/cache@v4\n id: parsers-cache\n with:\n path: build/lib\n key: edb-parsers-v3-${{ hashFiles('shared-artifacts/parsers_cache_key.txt') }}\n restore-keys: |\n edb-parsers-v3-\n\n - name: Build parsers\n env:\n CACHE_HIT: ${{ steps.parsers-cache.outputs.cache-hit }}\n run: |\n if [[ \"$CACHE_HIT\" != \"true\" ]]; then\n rm -rf ${BUILD_LIB}\n mkdir -p ./build/lib\n rsync -av ./build/lib/ ${BUILD_LIB}/\n python setup.py -v build_parsers\n rsync -av ${BUILD_LIB}/ ./build/lib/\n rm -rf ${BUILD_LIB}\n fi\n rsync -av ./build/lib/edb/ ./edb/\n\n # Build PostgreSQL\n\n - name: Build PostgreSQL\n env:\n CACHE_HIT: ${{ steps.postgres-cache.outputs.cache-hit }}\n run: |\n if [[ \"$CACHE_HIT\" == \"true\" ]]; then\n cp build/postgres/install/stamp build/postgres/\n else\n python setup.py build_postgres\n cp build/postgres/stamp build/postgres/install/\n fi\n\n # Build Stolon\n\n - name: Set up Go\n if: steps.stolon-cache.outputs.cache-hit != 'true'\n uses: actions/setup-go@v2\n with:\n go-version: 1.16\n\n - uses: actions/checkout@v4\n if: steps.stolon-cache.outputs.cache-hit != 'true'\n with:\n repository: edgedb/stolon\n path: build/stolon\n ref: ${{ env.STOLON_GIT_REV }}\n fetch-depth: 0\n submodules: false\n\n - name: Build Stolon\n if: steps.stolon-cache.outputs.cache-hit != 'true'\n run: |\n mkdir -p build/stolon/bin/\n curl -fsSL https://releases.hashicorp.com/consul/1.10.1/consul_1.10.1_linux_amd64.zip | zcat > build/stolon/bin/consul\n chmod +x build/stolon/bin/consul\n cd build/stolon && make\n\n # Install edgedb-server and populate egg-info\n\n - name: Install edgedb-server\n env:\n BUILD_EXT_MODE: skip\n run: |\n # --no-build-isolation because we have explicitly installed all deps\n # and don't want them to be reinstalled in an \"isolated env\".\n pip install --no-build-isolation --no-deps -e .[test,docs]\n\n # Refresh the bootstrap cache\n\n - name: Handle bootstrap cache\n uses: actions/cache@v4\n id: bootstrap-cache\n with:\n path: build/cache\n key: edb-bootstrap-v2-${{ hashFiles('shared-artifacts/bootstrap_cache_key.txt') }}\n restore-keys: |\n edb-bootstrap-v2-\n\n - name: Bootstrap EdgeDB Server\n if: steps.bootstrap-cache.outputs.cache-hit != 'true'\n run: |\n edb server --bootstrap-only\n\n\n setup-aws-rds:\n runs-on: ubuntu-latest\n outputs:\n pghost: ${{ steps.pghost.outputs.stdout }}\n defaults:\n run:\n working-directory: .github/aws-rds\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - name: Setup Terraform\n uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2.0.3\n\n - name: Initialize Terraform\n run: terraform init\n\n - name: Configure AWS Credentials\n uses: aws-actions/configure-aws-credentials@v1\n with:\n aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}\n aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}\n aws-region: us-east-2\n\n - name: Setup AWS RDS\n env:\n TF_VAR_sg_id: ${{ secrets.AWS_SECURITY_GROUP }}\n TF_VAR_password: ${{ secrets.AWS_RDS_PASSWORD }}\n run: |\n terraform apply -auto-approve\n\n - name: Store Terraform state\n if: ${{ always() }}\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: aws-rds-tfstate\n path: .github/aws-rds/terraform.tfstate\n retention-days: 1\n\n - name: Get RDS host\n id: pghost\n run: |\n terraform output -raw db_instance_address\n\n test-aws-rds:\n runs-on: ubuntu-latest\n needs: [setup-aws-rds, build]\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - uses: actions/checkout@v4\n with:\n fetch-depth: 50\n submodules: true\n\n - name: Set up Python\n uses: actions/setup-python@v5\n id: setup-python\n with:\n python-version: '3.12.2'\n cache: 'pip'\n cache-dependency-path: |\n pyproject.toml\n\n # The below is technically a lie as we are technically not\n # inside a virtual env, but there is really no reason to bother\n # actually creating and activating one as below works just fine.\n - name: Export $VIRTUAL_ENV\n run: |\n venv=\"$(python -c 'import sys; sys.stdout.write(sys.prefix)')\"\n echo \"VIRTUAL_ENV=${venv}\" >> $GITHUB_ENV\n\n - name: Set up uv cache\n uses: actions/cache@v4\n with:\n path: ~/.cache/uv\n key: uv-cache-${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}\n \n - name: Download requirements.txt\n uses: actions/cache@v4\n with:\n path: requirements.txt\n key: edb-requirements-${{ hashFiles('pyproject.toml') }}\n\n - name: Install Python dependencies\n run: |\n python -m pip install uv~=0.1.0 && uv pip install -U -r requirements.txt\n # 80.9.0 breaks our sphinx, and it keeps sneaking in\n uv pip install setuptools==80.8.0\n\n # Restore the artifacts and environment variables\n\n - name: Download shared artifacts\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: shared-artifacts\n path: shared-artifacts\n\n - name: Set environment variables\n run: |\n echo POSTGRES_GIT_REV=$(cat shared-artifacts/postgres_git_rev.txt) >> $GITHUB_ENV\n echo STOLON_GIT_REV=$(cat shared-artifacts/stolon_git_rev.txt) >> $GITHUB_ENV\n echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV\n echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV\n\n # Restore build cache\n\n - name: Restore cached Rust extensions\n uses: actions/cache@v4\n id: rust-cache\n with:\n path: build/rust_extensions\n key: edb-rust-v4-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n\n - name: Restore cached Cython extensions\n uses: actions/cache@v4\n id: ext-cache\n with:\n path: build/extensions\n key: edb-ext-v6-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Restore compiled parsers cache\n uses: actions/cache@v4\n id: parsers-cache\n with:\n path: build/lib\n key: edb-parsers-v3-${{ hashFiles('shared-artifacts/parsers_cache_key.txt') }}\n\n - name: Restore cached PostgreSQL build\n uses: actions/cache@v4\n id: postgres-cache\n with:\n path: build/postgres/install\n key: edb-postgres-v3-${{ env.POSTGRES_GIT_REV }}-${{ hashFiles('shared-artifacts/lib_cache_key.txt') }}\n\n - name: Restore cached Stolon build\n uses: actions/cache@v4\n id: stolon-cache\n with:\n path: build/stolon/bin\n key: edb-stolon-v2-${{ env.STOLON_GIT_REV }}\n\n - name: Restore bootstrap cache\n uses: actions/cache@v4\n id: bootstrap-cache\n with:\n path: build/cache\n key: edb-bootstrap-v2-${{ hashFiles('shared-artifacts/bootstrap_cache_key.txt') }}\n\n - name: Stop if we cannot retrieve the cache\n if: |\n steps.rust-cache.outputs.cache-hit != 'true' ||\n steps.ext-cache.outputs.cache-hit != 'true' ||\n steps.parsers-cache.outputs.cache-hit != 'true' ||\n steps.postgres-cache.outputs.cache-hit != 'true' ||\n steps.stolon-cache.outputs.cache-hit != 'true' ||\n steps.bootstrap-cache.outputs.cache-hit != 'true'\n run: |\n echo ::error::Cannot retrieve build cache.\n exit 1\n\n - name: Validate cached binaries\n run: |\n # Validate Stolon\n ./build/stolon/bin/stolon-sentinel --version || exit 1\n ./build/stolon/bin/stolon-keeper --version || exit 1\n ./build/stolon/bin/stolon-proxy --version || exit 1\n\n # Validate PostgreSQL\n ./build/postgres/install/bin/postgres --version || exit 1\n ./build/postgres/install/bin/pg_config --version || exit 1\n\n - name: Restore cache into the source tree\n run: |\n rsync -av ./build/rust_extensions/edb/ ./edb/\n rsync -av ./build/extensions/edb/ ./edb/\n rsync -av ./build/lib/edb/ ./edb/\n cp build/postgres/install/stamp build/postgres/\n\n - name: Install edgedb-server\n env:\n BUILD_EXT_MODE: skip\n run: |\n # --no-build-isolation because we have explicitly installed all deps\n # and don't want them to be reinstalled in an \"isolated env\".\n pip install --no-build-isolation --no-deps -e .[test,docs]\n\n # Run the test\n\n - name: Test\n env:\n EDGEDB_TEST_BACKEND_DSN: postgres://edbtest:${{ secrets.AWS_RDS_PASSWORD }}@${{ needs.setup-aws-rds.outputs.pghost }}/postgres\n run: |\n edb server --bootstrap-only --backend-dsn=$EDGEDB_TEST_BACKEND_DSN --testmode\n edb test -j2 -v --backend-dsn=$EDGEDB_TEST_BACKEND_DSN\n\n teardown-aws-rds:\n runs-on: ubuntu-latest\n needs: test-aws-rds\n if: ${{ always() }}\n defaults:\n run:\n working-directory: .github/aws-rds\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - name: Setup Terraform\n uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2.0.3\n\n - name: Initialize Terraform\n run: terraform init\n\n - name: Configure AWS Credentials\n uses: aws-actions/configure-aws-credentials@v1\n with:\n aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}\n aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}\n aws-region: us-east-2\n\n - name: Restore Terraform state\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: aws-rds-tfstate\n path: .github/aws-rds\n\n - name: Destroy AWS RDS\n run: terraform destroy -auto-approve\n env:\n TF_VAR_sg_id: ${{ secrets.AWS_SECURITY_GROUP }}\n TF_VAR_password: ${{ secrets.AWS_RDS_PASSWORD }}\n\n - name: Overwrite Terraform state\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: aws-rds-tfstate\n path: .github/aws-rds/terraform.tfstate\n retention-days: 1\n\n\n setup-do-database:\n runs-on: ubuntu-latest\n defaults:\n run:\n working-directory: .github/do-database\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - name: Setup Terraform\n uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2.0.3\n\n - name: Initialize Terraform\n run: terraform init\n\n - name: Setup DigitalOcean Database\n env:\n TF_VAR_do_token: ${{ secrets.DIGITALOCEAN_TOKEN }}\n run: |\n terraform apply -auto-approve\n\n - name: Store Terraform state\n if: ${{ always() }}\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: do-database-tfstate\n path: .github/do-database/terraform.tfstate\n retention-days: 1\n\n test-do-database:\n runs-on: ubuntu-latest\n needs: [setup-do-database, build]\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - uses: actions/checkout@v4\n with:\n fetch-depth: 50\n submodules: true\n\n - name: Set up Python\n uses: actions/setup-python@v5\n id: setup-python\n with:\n python-version: '3.12.2'\n cache: 'pip'\n cache-dependency-path: |\n pyproject.toml\n\n # The below is technically a lie as we are technically not\n # inside a virtual env, but there is really no reason to bother\n # actually creating and activating one as below works just fine.\n - name: Export $VIRTUAL_ENV\n run: |\n venv=\"$(python -c 'import sys; sys.stdout.write(sys.prefix)')\"\n echo \"VIRTUAL_ENV=${venv}\" >> $GITHUB_ENV\n\n - name: Set up uv cache\n uses: actions/cache@v4\n with:\n path: ~/.cache/uv\n key: uv-cache-${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}\n \n - name: Download requirements.txt\n uses: actions/cache@v4\n with:\n path: requirements.txt\n key: edb-requirements-${{ hashFiles('pyproject.toml') }}\n\n - name: Install Python dependencies\n run: |\n python -m pip install uv~=0.1.0 && uv pip install -U -r requirements.txt\n # 80.9.0 breaks our sphinx, and it keeps sneaking in\n uv pip install setuptools==80.8.0\n\n # Restore the artifacts and environment variables\n\n - name: Download shared artifacts\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: shared-artifacts\n path: shared-artifacts\n\n - name: Set environment variables\n run: |\n echo POSTGRES_GIT_REV=$(cat shared-artifacts/postgres_git_rev.txt) >> $GITHUB_ENV\n echo STOLON_GIT_REV=$(cat shared-artifacts/stolon_git_rev.txt) >> $GITHUB_ENV\n echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV\n echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV\n\n # Restore build cache\n\n - name: Restore cached Rust extensions\n uses: actions/cache@v4\n id: rust-cache\n with:\n path: build/rust_extensions\n key: edb-rust-v4-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n\n - name: Restore cached Cython extensions\n uses: actions/cache@v4\n id: ext-cache\n with:\n path: build/extensions\n key: edb-ext-v6-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Restore compiled parsers cache\n uses: actions/cache@v4\n id: parsers-cache\n with:\n path: build/lib\n key: edb-parsers-v3-${{ hashFiles('shared-artifacts/parsers_cache_key.txt') }}\n\n - name: Restore cached PostgreSQL build\n uses: actions/cache@v4\n id: postgres-cache\n with:\n path: build/postgres/install\n key: edb-postgres-v3-${{ env.POSTGRES_GIT_REV }}-${{ hashFiles('shared-artifacts/lib_cache_key.txt') }}\n\n - name: Restore cached Stolon build\n uses: actions/cache@v4\n id: stolon-cache\n with:\n path: build/stolon/bin\n key: edb-stolon-v2-${{ env.STOLON_GIT_REV }}\n\n - name: Restore bootstrap cache\n uses: actions/cache@v4\n id: bootstrap-cache\n with:\n path: build/cache\n key: edb-bootstrap-v2-${{ hashFiles('shared-artifacts/bootstrap_cache_key.txt') }}\n\n - name: Stop if we cannot retrieve the cache\n if: |\n steps.rust-cache.outputs.cache-hit != 'true' ||\n steps.ext-cache.outputs.cache-hit != 'true' ||\n steps.parsers-cache.outputs.cache-hit != 'true' ||\n steps.postgres-cache.outputs.cache-hit != 'true' ||\n steps.stolon-cache.outputs.cache-hit != 'true' ||\n steps.bootstrap-cache.outputs.cache-hit != 'true'\n run: |\n echo ::error::Cannot retrieve build cache.\n exit 1\n\n - name: Validate cached binaries\n run: |\n # Validate Stolon\n ./build/stolon/bin/stolon-sentinel --version || exit 1\n ./build/stolon/bin/stolon-keeper --version || exit 1\n ./build/stolon/bin/stolon-proxy --version || exit 1\n\n # Validate PostgreSQL\n ./build/postgres/install/bin/postgres --version || exit 1\n ./build/postgres/install/bin/pg_config --version || exit 1\n\n - name: Restore cache into the source tree\n run: |\n rsync -av ./build/rust_extensions/edb/ ./edb/\n rsync -av ./build/extensions/edb/ ./edb/\n rsync -av ./build/lib/edb/ ./edb/\n cp build/postgres/install/stamp build/postgres/\n\n - name: Install edgedb-server\n env:\n BUILD_EXT_MODE: skip\n run: |\n # --no-build-isolation because we have explicitly installed all deps\n # and don't want them to be reinstalled in an \"isolated env\".\n pip install --no-build-isolation --no-deps -e .[test,docs]\n\n - name: Setup Terraform\n uses: hashicorp/setup-terraform@v1\n\n - name: Initialize Terraform\n working-directory: .github/do-database\n run: terraform init\n\n - name: Restore Terraform state\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: do-database-tfstate\n path: .github/do-database\n\n - name: Get Database host\n id: pghost\n working-directory: .github/do-database\n run: |\n terraform output -raw db_instance_address\n\n - name: Get Database port\n id: pgport\n working-directory: .github/do-database\n run: |\n terraform output -raw db_instance_port\n\n - name: Get Database user\n id: pguser\n working-directory: .github/do-database\n run: |\n terraform output -raw db_instance_user\n\n - name: Get Database password\n id: pgpass\n working-directory: .github/do-database\n run: |\n terraform output -raw db_instance_password\n\n - name: Get Database dbname\n id: pgdatabase\n working-directory: .github/do-database\n run: |\n terraform output -raw db_instance_database\n\n # Run the test\n\n - name: Test\n env:\n EDGEDB_TEST_BACKEND_DSN: postgres://${{ steps.pguser.outputs.stdout }}:${{ steps.pgpass.outputs.stdout }}@${{ steps.pghost.outputs.stdout }}:${{ steps.pgport.outputs.stdout }}/${{ steps.pgdatabase.outputs.stdout }}\n run: |\n edb server --bootstrap-only --backend-dsn=$EDGEDB_TEST_BACKEND_DSN --testmode\n edb test -j2 -v --backend-dsn=$EDGEDB_TEST_BACKEND_DSN\n\n teardown-do-database:\n runs-on: ubuntu-latest\n needs: test-do-database\n if: ${{ always() }}\n defaults:\n run:\n working-directory: .github/do-database\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - name: Setup Terraform\n uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2.0.3\n\n - name: Initialize Terraform\n run: terraform init\n\n - name: Restore Terraform state\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: do-database-tfstate\n path: .github/do-database\n\n - name: Destroy DigitalOcean Database\n run: terraform destroy -auto-approve\n env:\n TF_VAR_do_token: ${{ secrets.DIGITALOCEAN_TOKEN }}\n\n - name: Overwrite Terraform state\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: do-database-tfstate\n path: .github/do-database/terraform.tfstate\n retention-days: 1\n\n\n setup-gcp-cloud-sql:\n runs-on: ubuntu-latest\n outputs:\n pghost: ${{ steps.pghost.outputs.stdout }}\n defaults:\n run:\n working-directory: .github/gcp-cloud-sql\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - name: Setup Terraform\n uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2.0.3\n\n - name: Initialize Terraform\n run: terraform init\n\n - name: Configure GCP Credentials\n uses: google-github-actions/setup-gcloud@main\n with:\n service_account_key: ${{ secrets.GCP_SA_KEY }}\n export_default_credentials: true\n\n - name: Setup GCP Cloud SQL\n env:\n TF_VAR_password: ${{ secrets.AWS_RDS_PASSWORD }}\n run: |\n terraform apply -auto-approve\n\n - name: Store Terraform state\n if: ${{ always() }}\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: gcp-cloud-sql-tfstate\n path: .github/gcp-cloud-sql/terraform.tfstate\n retention-days: 1\n\n - name: Get Cloud SQL host\n id: pghost\n run: |\n terraform output -raw db_instance_address\n\n test-gcp-cloud-sql:\n runs-on: ubuntu-latest\n needs: [setup-gcp-cloud-sql, build]\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - uses: actions/checkout@v4\n with:\n fetch-depth: 50\n submodules: true\n\n - name: Set up Python\n uses: actions/setup-python@v5\n id: setup-python\n with:\n python-version: '3.12.2'\n cache: 'pip'\n cache-dependency-path: |\n pyproject.toml\n\n # The below is technically a lie as we are technically not\n # inside a virtual env, but there is really no reason to bother\n # actually creating and activating one as below works just fine.\n - name: Export $VIRTUAL_ENV\n run: |\n venv=\"$(python -c 'import sys; sys.stdout.write(sys.prefix)')\"\n echo \"VIRTUAL_ENV=${venv}\" >> $GITHUB_ENV\n\n - name: Set up uv cache\n uses: actions/cache@v4\n with:\n path: ~/.cache/uv\n key: uv-cache-${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}\n \n - name: Download requirements.txt\n uses: actions/cache@v4\n with:\n path: requirements.txt\n key: edb-requirements-${{ hashFiles('pyproject.toml') }}\n\n - name: Install Python dependencies\n run: |\n python -m pip install uv~=0.1.0 && uv pip install -U -r requirements.txt\n # 80.9.0 breaks our sphinx, and it keeps sneaking in\n uv pip install setuptools==80.8.0\n\n # Restore the artifacts and environment variables\n\n - name: Download shared artifacts\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: shared-artifacts\n path: shared-artifacts\n\n - name: Set environment variables\n run: |\n echo POSTGRES_GIT_REV=$(cat shared-artifacts/postgres_git_rev.txt) >> $GITHUB_ENV\n echo STOLON_GIT_REV=$(cat shared-artifacts/stolon_git_rev.txt) >> $GITHUB_ENV\n echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV\n echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV\n\n # Restore build cache\n\n - name: Restore cached Rust extensions\n uses: actions/cache@v4\n id: rust-cache\n with:\n path: build/rust_extensions\n key: edb-rust-v4-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n\n - name: Restore cached Cython extensions\n uses: actions/cache@v4\n id: ext-cache\n with:\n path: build/extensions\n key: edb-ext-v6-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Restore compiled parsers cache\n uses: actions/cache@v4\n id: parsers-cache\n with:\n path: build/lib\n key: edb-parsers-v3-${{ hashFiles('shared-artifacts/parsers_cache_key.txt') }}\n\n - name: Restore cached PostgreSQL build\n uses: actions/cache@v4\n id: postgres-cache\n with:\n path: build/postgres/install\n key: edb-postgres-v3-${{ env.POSTGRES_GIT_REV }}-${{ hashFiles('shared-artifacts/lib_cache_key.txt') }}\n\n - name: Restore cached Stolon build\n uses: actions/cache@v4\n id: stolon-cache\n with:\n path: build/stolon/bin\n key: edb-stolon-v2-${{ env.STOLON_GIT_REV }}\n\n - name: Restore bootstrap cache\n uses: actions/cache@v4\n id: bootstrap-cache\n with:\n path: build/cache\n key: edb-bootstrap-v2-${{ hashFiles('shared-artifacts/bootstrap_cache_key.txt') }}\n\n - name: Stop if we cannot retrieve the cache\n if: |\n steps.rust-cache.outputs.cache-hit != 'true' ||\n steps.ext-cache.outputs.cache-hit != 'true' ||\n steps.parsers-cache.outputs.cache-hit != 'true' ||\n steps.postgres-cache.outputs.cache-hit != 'true' ||\n steps.stolon-cache.outputs.cache-hit != 'true' ||\n steps.bootstrap-cache.outputs.cache-hit != 'true'\n run: |\n echo ::error::Cannot retrieve build cache.\n exit 1\n\n - name: Validate cached binaries\n run: |\n # Validate Stolon\n ./build/stolon/bin/stolon-sentinel --version || exit 1\n ./build/stolon/bin/stolon-keeper --version || exit 1\n ./build/stolon/bin/stolon-proxy --version || exit 1\n\n # Validate PostgreSQL\n ./build/postgres/install/bin/postgres --version || exit 1\n ./build/postgres/install/bin/pg_config --version || exit 1\n\n - name: Restore cache into the source tree\n run: |\n rsync -av ./build/rust_extensions/edb/ ./edb/\n rsync -av ./build/extensions/edb/ ./edb/\n rsync -av ./build/lib/edb/ ./edb/\n cp build/postgres/install/stamp build/postgres/\n\n - name: Install edgedb-server\n env:\n BUILD_EXT_MODE: skip\n run: |\n # --no-build-isolation because we have explicitly installed all deps\n # and don't want them to be reinstalled in an \"isolated env\".\n pip install --no-build-isolation --no-deps -e .[test,docs]\n\n # Run the test\n\n - name: Test\n env:\n EDGEDB_TEST_BACKEND_DSN: postgres://postgres:${{ secrets.AWS_RDS_PASSWORD }}@${{ needs.setup-gcp-cloud-sql.outputs.pghost }}/postgres\n run: |\n edb server --bootstrap-only --backend-dsn=$EDGEDB_TEST_BACKEND_DSN --testmode\n edb test -j2 -v --backend-dsn=$EDGEDB_TEST_BACKEND_DSN\n\n teardown-gcp-cloud-sql:\n runs-on: ubuntu-latest\n needs: test-gcp-cloud-sql\n if: ${{ always() }}\n defaults:\n run:\n working-directory: .github/gcp-cloud-sql\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - name: Setup Terraform\n uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2.0.3\n\n - name: Initialize Terraform\n run: terraform init\n\n - name: Configure GCP Credentials\n uses: google-github-actions/setup-gcloud@main\n with:\n service_account_key: ${{ secrets.GCP_SA_KEY }}\n export_default_credentials: true\n\n - name: Restore Terraform state\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: gcp-cloud-sql-tfstate\n path: .github/gcp-cloud-sql\n\n - name: Destroy GCP Cloud SQL\n run: terraform destroy -auto-approve\n env:\n TF_VAR_password: ${{ secrets.AWS_RDS_PASSWORD }}\n\n - name: Overwrite Terraform state\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: gcp-cloud-sql-tfstate\n path: .github/gcp-cloud-sql/terraform.tfstate\n retention-days: 1\n\n\n setup-aws-aurora:\n runs-on: ubuntu-latest\n outputs:\n pghost: ${{ steps.pghost.outputs.stdout }}\n defaults:\n run:\n working-directory: .github/aws-aurora\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - name: Setup Terraform\n uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2.0.3\n\n - name: Initialize Terraform\n run: terraform init\n\n - name: Configure AWS Credentials\n uses: aws-actions/configure-aws-credentials@v1\n with:\n aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}\n aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}\n aws-region: us-east-2\n\n - name: Setup AWS RDS Aurora\n env:\n TF_VAR_sg_id: ${{ secrets.AWS_SECURITY_GROUP }}\n TF_VAR_password: ${{ secrets.AWS_RDS_PASSWORD }}\n TF_VAR_vpc_id: ${{ secrets.AWS_VPC_ID }}\n run: |\n terraform apply -auto-approve\n\n - name: Store Terraform state\n if: ${{ always() }}\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: aws-aurora-tfstate\n path: .github/aws-aurora/terraform.tfstate\n retention-days: 1\n\n - name: Get RDS Aurora host\n id: pghost\n run: |\n terraform output -raw rds_cluster_endpoint\n\n test-aws-aurora:\n runs-on: ubuntu-latest\n needs: [setup-aws-aurora, build]\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - uses: actions/checkout@v4\n with:\n fetch-depth: 50\n submodules: true\n\n - name: Set up Python\n uses: actions/setup-python@v5\n id: setup-python\n with:\n python-version: '3.12.2'\n cache: 'pip'\n cache-dependency-path: |\n pyproject.toml\n\n # The below is technically a lie as we are technically not\n # inside a virtual env, but there is really no reason to bother\n # actually creating and activating one as below works just fine.\n - name: Export $VIRTUAL_ENV\n run: |\n venv=\"$(python -c 'import sys; sys.stdout.write(sys.prefix)')\"\n echo \"VIRTUAL_ENV=${venv}\" >> $GITHUB_ENV\n\n - name: Set up uv cache\n uses: actions/cache@v4\n with:\n path: ~/.cache/uv\n key: uv-cache-${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}\n \n - name: Download requirements.txt\n uses: actions/cache@v4\n with:\n path: requirements.txt\n key: edb-requirements-${{ hashFiles('pyproject.toml') }}\n\n - name: Install Python dependencies\n run: |\n python -m pip install uv~=0.1.0 && uv pip install -U -r requirements.txt\n # 80.9.0 breaks our sphinx, and it keeps sneaking in\n uv pip install setuptools==80.8.0\n\n # Restore the artifacts and environment variables\n\n - name: Download shared artifacts\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: shared-artifacts\n path: shared-artifacts\n\n - name: Set environment variables\n run: |\n echo POSTGRES_GIT_REV=$(cat shared-artifacts/postgres_git_rev.txt) >> $GITHUB_ENV\n echo STOLON_GIT_REV=$(cat shared-artifacts/stolon_git_rev.txt) >> $GITHUB_ENV\n echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV\n echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV\n\n # Restore build cache\n\n - name: Restore cached Rust extensions\n uses: actions/cache@v4\n id: rust-cache\n with:\n path: build/rust_extensions\n key: edb-rust-v4-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n\n - name: Restore cached Cython extensions\n uses: actions/cache@v4\n id: ext-cache\n with:\n path: build/extensions\n key: edb-ext-v6-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Restore compiled parsers cache\n uses: actions/cache@v4\n id: parsers-cache\n with:\n path: build/lib\n key: edb-parsers-v3-${{ hashFiles('shared-artifacts/parsers_cache_key.txt') }}\n\n - name: Restore cached PostgreSQL build\n uses: actions/cache@v4\n id: postgres-cache\n with:\n path: build/postgres/install\n key: edb-postgres-v3-${{ env.POSTGRES_GIT_REV }}-${{ hashFiles('shared-artifacts/lib_cache_key.txt') }}\n\n - name: Restore cached Stolon build\n uses: actions/cache@v4\n id: stolon-cache\n with:\n path: build/stolon/bin\n key: edb-stolon-v2-${{ env.STOLON_GIT_REV }}\n\n - name: Restore bootstrap cache\n uses: actions/cache@v4\n id: bootstrap-cache\n with:\n path: build/cache\n key: edb-bootstrap-v2-${{ hashFiles('shared-artifacts/bootstrap_cache_key.txt') }}\n\n - name: Stop if we cannot retrieve the cache\n if: |\n steps.rust-cache.outputs.cache-hit != 'true' ||\n steps.ext-cache.outputs.cache-hit != 'true' ||\n steps.parsers-cache.outputs.cache-hit != 'true' ||\n steps.postgres-cache.outputs.cache-hit != 'true' ||\n steps.stolon-cache.outputs.cache-hit != 'true' ||\n steps.bootstrap-cache.outputs.cache-hit != 'true'\n run: |\n echo ::error::Cannot retrieve build cache.\n exit 1\n\n - name: Validate cached binaries\n run: |\n # Validate Stolon\n ./build/stolon/bin/stolon-sentinel --version || exit 1\n ./build/stolon/bin/stolon-keeper --version || exit 1\n ./build/stolon/bin/stolon-proxy --version || exit 1\n\n # Validate PostgreSQL\n ./build/postgres/install/bin/postgres --version || exit 1\n ./build/postgres/install/bin/pg_config --version || exit 1\n\n - name: Restore cache into the source tree\n run: |\n rsync -av ./build/rust_extensions/edb/ ./edb/\n rsync -av ./build/extensions/edb/ ./edb/\n rsync -av ./build/lib/edb/ ./edb/\n cp build/postgres/install/stamp build/postgres/\n\n - name: Install edgedb-server\n env:\n BUILD_EXT_MODE: skip\n run: |\n # --no-build-isolation because we have explicitly installed all deps\n # and don't want them to be reinstalled in an \"isolated env\".\n pip install --no-build-isolation --no-deps -e .[test,docs]\n\n # Run the test\n\n - name: Test\n env:\n EDGEDB_TEST_BACKEND_DSN: postgres://edbtest:${{ secrets.AWS_RDS_PASSWORD }}@${{ needs.setup-aws-aurora.outputs.pghost }}/postgres\n run: |\n edb server --bootstrap-only --backend-dsn=$EDGEDB_TEST_BACKEND_DSN --testmode\n edb test -j1 -v --backend-dsn=$EDGEDB_TEST_BACKEND_DSN\n\n teardown-aws-aurora:\n runs-on: ubuntu-latest\n needs: test-aws-aurora\n if: ${{ always() }}\n defaults:\n run:\n working-directory: .github/aws-aurora\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - name: Setup Terraform\n uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2.0.3\n\n - name: Initialize Terraform\n run: terraform init\n\n - name: Configure AWS Credentials\n uses: aws-actions/configure-aws-credentials@v1\n with:\n aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}\n aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}\n aws-region: us-east-2\n\n - name: Restore Terraform state\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: aws-aurora-tfstate\n path: .github/aws-aurora\n\n - name: Destroy AWS RDS Aurora\n run: terraform destroy -auto-approve\n env:\n TF_VAR_sg_id: ${{ secrets.AWS_SECURITY_GROUP }}\n TF_VAR_password: ${{ secrets.AWS_RDS_PASSWORD }}\n TF_VAR_vpc_id: ${{ secrets.AWS_VPC_ID }}\n\n - name: Overwrite Terraform state\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: aws-aurora-tfstate\n path: .github/aws-aurora/terraform.tfstate\n retention-days: 1\n\n\n setup-heroku-postgres:\n runs-on: ubuntu-latest\n defaults:\n run:\n working-directory: .github/heroku-postgres\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - name: Setup Terraform\n uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2.0.3\n\n - name: Initialize Terraform\n run: terraform init\n\n - name: Setup Heroku Postgres\n env:\n HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}\n HEROKU_EMAIL: ${{ secrets.HEROKU_EMAIL }}\n run: |\n terraform apply -auto-approve\n\n - name: Store Terraform state\n if: ${{ always() }}\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: heroku-postgres-tfstate\n path: .github/heroku-postgres/terraform.tfstate\n retention-days: 1\n\n test-heroku-postgres:\n runs-on: ubuntu-latest\n needs: [setup-heroku-postgres, build]\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - uses: actions/checkout@v4\n with:\n fetch-depth: 50\n submodules: true\n\n - name: Set up Python\n uses: actions/setup-python@v5\n id: setup-python\n with:\n python-version: '3.12.2'\n cache: 'pip'\n cache-dependency-path: |\n pyproject.toml\n\n # The below is technically a lie as we are technically not\n # inside a virtual env, but there is really no reason to bother\n # actually creating and activating one as below works just fine.\n - name: Export $VIRTUAL_ENV\n run: |\n venv=\"$(python -c 'import sys; sys.stdout.write(sys.prefix)')\"\n echo \"VIRTUAL_ENV=${venv}\" >> $GITHUB_ENV\n\n - name: Set up uv cache\n uses: actions/cache@v4\n with:\n path: ~/.cache/uv\n key: uv-cache-${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}\n \n - name: Download requirements.txt\n uses: actions/cache@v4\n with:\n path: requirements.txt\n key: edb-requirements-${{ hashFiles('pyproject.toml') }}\n\n - name: Install Python dependencies\n run: |\n python -m pip install uv~=0.1.0 && uv pip install -U -r requirements.txt\n # 80.9.0 breaks our sphinx, and it keeps sneaking in\n uv pip install setuptools==80.8.0\n\n # Restore the artifacts and environment variables\n\n - name: Download shared artifacts\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: shared-artifacts\n path: shared-artifacts\n\n - name: Set environment variables\n run: |\n echo POSTGRES_GIT_REV=$(cat shared-artifacts/postgres_git_rev.txt) >> $GITHUB_ENV\n echo STOLON_GIT_REV=$(cat shared-artifacts/stolon_git_rev.txt) >> $GITHUB_ENV\n echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV\n echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV\n\n # Restore build cache\n\n - name: Restore cached Rust extensions\n uses: actions/cache@v4\n id: rust-cache\n with:\n path: build/rust_extensions\n key: edb-rust-v4-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n\n - name: Restore cached Cython extensions\n uses: actions/cache@v4\n id: ext-cache\n with:\n path: build/extensions\n key: edb-ext-v6-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Restore compiled parsers cache\n uses: actions/cache@v4\n id: parsers-cache\n with:\n path: build/lib\n key: edb-parsers-v3-${{ hashFiles('shared-artifacts/parsers_cache_key.txt') }}\n\n - name: Restore cached PostgreSQL build\n uses: actions/cache@v4\n id: postgres-cache\n with:\n path: build/postgres/install\n key: edb-postgres-v3-${{ env.POSTGRES_GIT_REV }}-${{ hashFiles('shared-artifacts/lib_cache_key.txt') }}\n\n - name: Restore cached Stolon build\n uses: actions/cache@v4\n id: stolon-cache\n with:\n path: build/stolon/bin\n key: edb-stolon-v2-${{ env.STOLON_GIT_REV }}\n\n - name: Restore bootstrap cache\n uses: actions/cache@v4\n id: bootstrap-cache\n with:\n path: build/cache\n key: edb-bootstrap-v2-${{ hashFiles('shared-artifacts/bootstrap_cache_key.txt') }}\n\n - name: Stop if we cannot retrieve the cache\n if: |\n steps.rust-cache.outputs.cache-hit != 'true' ||\n steps.ext-cache.outputs.cache-hit != 'true' ||\n steps.parsers-cache.outputs.cache-hit != 'true' ||\n steps.postgres-cache.outputs.cache-hit != 'true' ||\n steps.stolon-cache.outputs.cache-hit != 'true' ||\n steps.bootstrap-cache.outputs.cache-hit != 'true'\n run: |\n echo ::error::Cannot retrieve build cache.\n exit 1\n\n - name: Validate cached binaries\n run: |\n # Validate Stolon\n ./build/stolon/bin/stolon-sentinel --version || exit 1\n ./build/stolon/bin/stolon-keeper --version || exit 1\n ./build/stolon/bin/stolon-proxy --version || exit 1\n\n # Validate PostgreSQL\n ./build/postgres/install/bin/postgres --version || exit 1\n ./build/postgres/install/bin/pg_config --version || exit 1\n\n - name: Restore cache into the source tree\n run: |\n rsync -av ./build/rust_extensions/edb/ ./edb/\n rsync -av ./build/extensions/edb/ ./edb/\n rsync -av ./build/lib/edb/ ./edb/\n cp build/postgres/install/stamp build/postgres/\n\n - name: Install edgedb-server\n env:\n BUILD_EXT_MODE: skip\n run: |\n # --no-build-isolation because we have explicitly installed all deps\n # and don't want them to be reinstalled in an \"isolated env\".\n pip install --no-build-isolation --no-deps -e .[test,docs]\n\n - name: Setup Terraform\n uses: hashicorp/setup-terraform@v1\n\n - name: Initialize Terraform\n working-directory: .github/heroku-postgres\n run: terraform init\n\n - name: Restore Terraform state\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: heroku-postgres-tfstate\n path: .github/heroku-postgres\n\n - name: Get Heroku Postgres DSN\n id: pgdsn\n working-directory: .github/heroku-postgres\n run: |\n terraform output -raw heroku_postgres_dsn\n\n # Run the test\n\n - name: Test\n env:\n EDGEDB_TEST_BACKEND_VENDOR: heroku-postgres\n EDGEDB_TEST_BACKEND_DSN: ${{ steps.pgdsn.outputs.stdout }}\n run: |\n edb server --bootstrap-only --backend-dsn=$EDGEDB_TEST_BACKEND_DSN --testmode\n edb test -j1 -v --backend-dsn=$EDGEDB_TEST_BACKEND_DSN\n\n teardown-heroku-postgres:\n runs-on: ubuntu-latest\n needs: test-heroku-postgres\n if: ${{ always() }}\n defaults:\n run:\n working-directory: .github/heroku-postgres\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - name: Setup Terraform\n uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2.0.3\n\n - name: Initialize Terraform\n run: terraform init\n\n - name: Configure AWS Credentials\n uses: aws-actions/configure-aws-credentials@v1\n with:\n aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}\n aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}\n aws-region: us-east-2\n\n - name: Restore Terraform state\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: heroku-postgres-tfstate\n path: .github/heroku-postgres\n\n - name: Destroy Heroku Postgres\n run: terraform destroy -auto-approve\n env:\n HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}\n HEROKU_EMAIL: ${{ secrets.HEROKU_EMAIL }}\n\n - name: Overwrite Terraform state\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: heroku-postgres-tfstate\n path: .github/heroku-postgres/terraform.tfstate\n retention-days: 1\n\n\n workflow-notifications:\n if: failure() && github.event_name != 'pull_request'\n name: Notify in Slack on failures\n needs:\n - setup-aws-rds\n - test-aws-rds\n - teardown-aws-rds\n - setup-do-database\n - test-do-database\n - teardown-do-database\n - setup-gcp-cloud-sql\n - test-gcp-cloud-sql\n - teardown-gcp-cloud-sql\n - setup-aws-aurora\n - test-aws-aurora\n - teardown-aws-aurora\n - setup-heroku-postgres\n - test-heroku-postgres\n - teardown-heroku-postgres\n runs-on: ubuntu-latest\n permissions:\n actions: 'read'\n steps:\n - name: Slack Workflow Notification\n uses: Gamesight/slack-workflow-status@26a36836c887f260477432e4314ec3490a84f309\n with:\n repo_token: ${{secrets.GITHUB_TOKEN}}\n slack_webhook_url: ${{secrets.ACTIONS_SLACK_WEBHOOK_URL}}\n name: 'Workflow notifications'\n icon_emoji: ':hammer:'\n include_jobs: 'on-failure'\n" }, { "path": ".github/workflows/tests.patches.yml", "content": "name: Tests of patching old EdgeDB Versions\n\non:\n workflow_dispatch:\n inputs: {}\n pull_request:\n branches:\n - release/*\n push:\n branches:\n - patch-test*\n - release/*\n\njobs:\n build:\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - uses: actions/checkout@v4\n with:\n fetch-depth: 50\n submodules: true\n\n - name: Set up Python\n uses: actions/setup-python@v5\n id: setup-python\n with:\n python-version: '3.12.2'\n cache: 'pip'\n cache-dependency-path: |\n pyproject.toml\n\n # The below is technically a lie as we are technically not\n # inside a virtual env, but there is really no reason to bother\n # actually creating and activating one as below works just fine.\n - name: Export $VIRTUAL_ENV\n run: |\n venv=\"$(python -c 'import sys; sys.stdout.write(sys.prefix)')\"\n echo \"VIRTUAL_ENV=${venv}\" >> $GITHUB_ENV\n\n - name: Set up uv cache\n uses: actions/cache@v4\n with:\n path: ~/.cache/uv\n key: uv-cache-${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}\n\n - name: Cached requirements.txt\n uses: actions/cache@v4\n id: requirements-cache\n with:\n path: requirements.txt\n key: edb-requirements-${{ hashFiles('pyproject.toml') }}\n\n - name: Compute requirements.txt\n if: steps.requirements-cache.outputs.cache-hit != 'true'\n run: |\n python -m pip install pip-tools\n pip-compile --no-strip-extras --all-build-deps \\\n --extra test,language-server \\\n --output-file requirements.txt pyproject.toml\n\n - name: Install Python dependencies\n run: |\n python -c \"import sys; print(sys.prefix)\"\n python -m pip install uv~=0.1.0 && uv pip install -U -r requirements.txt\n # 80.9.0 breaks our sphinx, and it keeps sneaking in\n uv pip install setuptools==80.8.0\n\n - name: Compute cache keys\n run: |\n mkdir -p shared-artifacts\n if [ \"$(uname)\" = \"Darwin\" ]; then\n find /usr/lib -type f -name 'lib*' -exec stat -f '%N %z' {} + | sort | shasum -a 256 | cut -d ' ' -f1 > shared-artifacts/lib_cache_key.txt\n else\n find /usr/lib -type f -name 'lib*' -printf '%P %s\\n' | sort | sha256sum | cut -d ' ' -f1 > shared-artifacts/lib_cache_key.txt\n fi\n python setup.py -q ci_helper --type rust >shared-artifacts/rust_cache_key.txt\n python setup.py -q ci_helper --type ext >shared-artifacts/ext_cache_key.txt\n python setup.py -q ci_helper --type parsers >shared-artifacts/parsers_cache_key.txt\n python setup.py -q ci_helper --type postgres >shared-artifacts/postgres_git_rev.txt\n python setup.py -q ci_helper --type libpg_query >shared-artifacts/libpg_query_git_rev.txt\n echo 'f8cd94309eaccbfba5dea7835b88c78377608a37' >shared-artifacts/stolon_git_rev.txt\n python setup.py -q ci_helper --type bootstrap >shared-artifacts/bootstrap_cache_key.txt\n echo POSTGRES_GIT_REV=$(cat shared-artifacts/postgres_git_rev.txt) >> $GITHUB_ENV\n echo LIBPG_QUERY_GIT_REV=$(cat shared-artifacts/libpg_query_git_rev.txt) >> $GITHUB_ENV\n echo STOLON_GIT_REV=$(cat shared-artifacts/stolon_git_rev.txt) >> $GITHUB_ENV\n echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV\n echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV\n\n - name: Upload shared artifacts\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: shared-artifacts\n path: shared-artifacts\n retention-days: 1\n\n # Restore binary cache\n\n - name: Handle cached Rust extensions\n uses: actions/cache@v4\n id: rust-cache\n with:\n path: build/rust_extensions\n key: edb-rust-v4-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n restore-keys: |\n edb-rust-v4-\n\n - name: Handle cached Cython extensions\n uses: actions/cache@v4\n id: ext-cache\n with:\n path: build/extensions\n key: edb-ext-v6-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Handle cached PostgreSQL build\n uses: actions/cache@v4\n id: postgres-cache\n with:\n path: build/postgres/install\n key: edb-postgres-v3-${{ env.POSTGRES_GIT_REV }}-${{ hashFiles('shared-artifacts/lib_cache_key.txt') }}\n\n - name: Handle cached Stolon build\n uses: actions/cache@v4\n id: stolon-cache\n with:\n path: build/stolon/bin\n key: edb-stolon-v2-${{ env.STOLON_GIT_REV }}\n\n - name: Handle cached libpg_query build\n uses: actions/cache@v4\n id: libpg-query-cache\n with:\n path: edb/pgsql/parser/libpg_query/libpg_query.a\n key: edb-libpg_query-v1-${{ env.LIBPG_QUERY_GIT_REV }}\n\n # Install system dependencies for building\n\n - name: Install system deps\n if: |\n steps.rust-cache.outputs.cache-hit != 'true' ||\n steps.ext-cache.outputs.cache-hit != 'true' ||\n steps.stolon-cache.outputs.cache-hit != 'true' ||\n steps.postgres-cache.outputs.cache-hit != 'true'\n run: |\n sudo apt-get update\n sudo apt-get install -y uuid-dev libreadline-dev bison flex libprotobuf-c-dev\n\n - name: Install Rust toolchain\n if: steps.rust-cache.outputs.cache-hit != 'true'\n uses: dsherret/rust-toolchain-file@v1\n\n # Build Rust extensions\n\n - name: Handle Rust extensions build cache\n uses: actions/cache@v4\n if: steps.rust-cache.outputs.cache-hit != 'true'\n with:\n path: ${{ env.BUILD_TEMP }}/rust/extensions\n key: edb-rust-build-v1-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n restore-keys: |\n edb-rust-build-v1-\n\n - name: Build Rust extensions\n env:\n CARGO_HOME: ${{ env.BUILD_TEMP }}/rust/extensions/cargo_home\n CACHE_HIT: ${{ steps.rust-cache.outputs.cache-hit }}\n run: |\n if [[ \"$CACHE_HIT\" != \"true\" ]]; then\n rm -rf ${BUILD_LIB}\n mkdir -p build/rust_extensions\n rsync -av ./build/rust_extensions/ ${BUILD_LIB}/\n python setup.py -v build_rust\n rsync -av ${BUILD_LIB}/ build/rust_extensions/\n rm -rf ${BUILD_LIB}\n fi\n rsync -av ./build/rust_extensions/edb/ ./edb/\n\n # Build libpg_query\n\n - name: Build libpg_query\n if: |\n steps.libpg-query-cache.outputs.cache-hit != 'true' &&\n steps.ext-cache.outputs.cache-hit != 'true'\n run: |\n python setup.py build_libpg_query\n\n # Build extensions\n\n - name: Handle Cython extensions build cache\n uses: actions/cache@v4\n if: steps.ext-cache.outputs.cache-hit != 'true'\n with:\n path: ${{ env.BUILD_TEMP }}/edb\n key: edb-ext-build-v4-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Build Cython extensions\n env:\n CACHE_HIT: ${{ steps.ext-cache.outputs.cache-hit }}\n BUILD_EXT_MODE: py-only\n run: |\n if [[ \"$CACHE_HIT\" != \"true\" ]]; then\n rm -rf ${BUILD_LIB}\n mkdir -p ./build/extensions\n rsync -av ./build/extensions/ ${BUILD_LIB}/\n BUILD_EXT_MODE=py-only python setup.py -v build_ext\n rsync -av ${BUILD_LIB}/ ./build/extensions/\n rm -rf ${BUILD_LIB}\n fi\n rsync -av ./build/extensions/edb/ ./edb/\n\n # Build parsers\n\n - name: Handle compiled parsers cache\n uses: actions/cache@v4\n id: parsers-cache\n with:\n path: build/lib\n key: edb-parsers-v3-${{ hashFiles('shared-artifacts/parsers_cache_key.txt') }}\n restore-keys: |\n edb-parsers-v3-\n\n - name: Build parsers\n env:\n CACHE_HIT: ${{ steps.parsers-cache.outputs.cache-hit }}\n run: |\n if [[ \"$CACHE_HIT\" != \"true\" ]]; then\n rm -rf ${BUILD_LIB}\n mkdir -p ./build/lib\n rsync -av ./build/lib/ ${BUILD_LIB}/\n python setup.py -v build_parsers\n rsync -av ${BUILD_LIB}/ ./build/lib/\n rm -rf ${BUILD_LIB}\n fi\n rsync -av ./build/lib/edb/ ./edb/\n\n # Build PostgreSQL\n\n - name: Build PostgreSQL\n env:\n CACHE_HIT: ${{ steps.postgres-cache.outputs.cache-hit }}\n run: |\n if [[ \"$CACHE_HIT\" == \"true\" ]]; then\n cp build/postgres/install/stamp build/postgres/\n else\n python setup.py build_postgres\n cp build/postgres/stamp build/postgres/install/\n fi\n\n # Build Stolon\n\n - name: Set up Go\n if: steps.stolon-cache.outputs.cache-hit != 'true'\n uses: actions/setup-go@v2\n with:\n go-version: 1.16\n\n - uses: actions/checkout@v4\n if: steps.stolon-cache.outputs.cache-hit != 'true'\n with:\n repository: edgedb/stolon\n path: build/stolon\n ref: ${{ env.STOLON_GIT_REV }}\n fetch-depth: 0\n submodules: false\n\n - name: Build Stolon\n if: steps.stolon-cache.outputs.cache-hit != 'true'\n run: |\n mkdir -p build/stolon/bin/\n curl -fsSL https://releases.hashicorp.com/consul/1.10.1/consul_1.10.1_linux_amd64.zip | zcat > build/stolon/bin/consul\n chmod +x build/stolon/bin/consul\n cd build/stolon && make\n\n # Install edgedb-server and populate egg-info\n\n - name: Install edgedb-server\n env:\n BUILD_EXT_MODE: skip\n run: |\n # --no-build-isolation because we have explicitly installed all deps\n # and don't want them to be reinstalled in an \"isolated env\".\n pip install --no-build-isolation --no-deps -e .[test,docs]\n\n # Refresh the bootstrap cache\n\n - name: Handle bootstrap cache\n uses: actions/cache@v4\n id: bootstrap-cache\n with:\n path: build/cache\n key: edb-bootstrap-v2-${{ hashFiles('shared-artifacts/bootstrap_cache_key.txt') }}\n restore-keys: |\n edb-bootstrap-v2-\n\n - name: Bootstrap EdgeDB Server\n if: steps.bootstrap-cache.outputs.cache-hit != 'true'\n run: |\n edb server --bootstrap-only\n\n compute-versions:\n runs-on: ubuntu-latest\n outputs:\n matrix: ${{ steps.set-matrix.outputs.matrix }}\n steps:\n - uses: actions/checkout@v4\n - id: set-matrix\n name: Compute versions to run on\n run: python3 .github/scripts/patches/compute-versions.py\n\n test:\n runs-on: ubuntu-latest\n needs: [build, compute-versions]\n strategy:\n fail-fast: false\n matrix: ${{fromJSON(needs.compute-versions.outputs.matrix)}}\n\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - uses: actions/checkout@v4\n with:\n fetch-depth: 50\n submodules: true\n\n - name: Set up Python\n uses: actions/setup-python@v5\n id: setup-python\n with:\n python-version: '3.12.2'\n cache: 'pip'\n cache-dependency-path: |\n pyproject.toml\n\n # The below is technically a lie as we are technically not\n # inside a virtual env, but there is really no reason to bother\n # actually creating and activating one as below works just fine.\n - name: Export $VIRTUAL_ENV\n run: |\n venv=\"$(python -c 'import sys; sys.stdout.write(sys.prefix)')\"\n echo \"VIRTUAL_ENV=${venv}\" >> $GITHUB_ENV\n\n - name: Set up uv cache\n uses: actions/cache@v4\n with:\n path: ~/.cache/uv\n key: uv-cache-${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}\n \n - name: Download requirements.txt\n uses: actions/cache@v4\n with:\n path: requirements.txt\n key: edb-requirements-${{ hashFiles('pyproject.toml') }}\n\n - name: Install Python dependencies\n run: |\n python -m pip install uv~=0.1.0 && uv pip install -U -r requirements.txt\n # 80.9.0 breaks our sphinx, and it keeps sneaking in\n uv pip install setuptools==80.8.0\n\n # Restore the artifacts and environment variables\n\n - name: Download shared artifacts\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: shared-artifacts\n path: shared-artifacts\n\n - name: Set environment variables\n run: |\n echo POSTGRES_GIT_REV=$(cat shared-artifacts/postgres_git_rev.txt) >> $GITHUB_ENV\n echo STOLON_GIT_REV=$(cat shared-artifacts/stolon_git_rev.txt) >> $GITHUB_ENV\n echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV\n echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV\n\n # Restore build cache\n\n - name: Restore cached Rust extensions\n uses: actions/cache@v4\n id: rust-cache\n with:\n path: build/rust_extensions\n key: edb-rust-v4-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n\n - name: Restore cached Cython extensions\n uses: actions/cache@v4\n id: ext-cache\n with:\n path: build/extensions\n key: edb-ext-v6-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Restore compiled parsers cache\n uses: actions/cache@v4\n id: parsers-cache\n with:\n path: build/lib\n key: edb-parsers-v3-${{ hashFiles('shared-artifacts/parsers_cache_key.txt') }}\n\n - name: Restore cached PostgreSQL build\n uses: actions/cache@v4\n id: postgres-cache\n with:\n path: build/postgres/install\n key: edb-postgres-v3-${{ env.POSTGRES_GIT_REV }}-${{ hashFiles('shared-artifacts/lib_cache_key.txt') }}\n\n - name: Restore cached Stolon build\n uses: actions/cache@v4\n id: stolon-cache\n with:\n path: build/stolon/bin\n key: edb-stolon-v2-${{ env.STOLON_GIT_REV }}\n\n - name: Restore bootstrap cache\n uses: actions/cache@v4\n id: bootstrap-cache\n with:\n path: build/cache\n key: edb-bootstrap-v2-${{ hashFiles('shared-artifacts/bootstrap_cache_key.txt') }}\n\n - name: Stop if we cannot retrieve the cache\n if: |\n steps.rust-cache.outputs.cache-hit != 'true' ||\n steps.ext-cache.outputs.cache-hit != 'true' ||\n steps.parsers-cache.outputs.cache-hit != 'true' ||\n steps.postgres-cache.outputs.cache-hit != 'true' ||\n steps.stolon-cache.outputs.cache-hit != 'true' ||\n steps.bootstrap-cache.outputs.cache-hit != 'true'\n run: |\n echo ::error::Cannot retrieve build cache.\n exit 1\n\n - name: Validate cached binaries\n run: |\n # Validate Stolon\n ./build/stolon/bin/stolon-sentinel --version || exit 1\n ./build/stolon/bin/stolon-keeper --version || exit 1\n ./build/stolon/bin/stolon-proxy --version || exit 1\n\n # Validate PostgreSQL\n ./build/postgres/install/bin/postgres --version || exit 1\n ./build/postgres/install/bin/pg_config --version || exit 1\n\n - name: Restore cache into the source tree\n run: |\n rsync -av ./build/rust_extensions/edb/ ./edb/\n rsync -av ./build/extensions/edb/ ./edb/\n rsync -av ./build/lib/edb/ ./edb/\n cp build/postgres/install/stamp build/postgres/\n\n - name: Install edgedb-server\n env:\n BUILD_EXT_MODE: skip\n run: |\n # --no-build-isolation because we have explicitly installed all deps\n # and don't want them to be reinstalled in an \"isolated env\".\n pip install --no-build-isolation --no-deps -e .[test,docs]\n\n # Run the test\n\n - name: Download an earlier database version and set up a instance\n run: |\n wget -q \"${{ matrix.edgedb-url }}\"\n tar xzf ${{ matrix.edgedb-basename }}-${{ matrix.edgedb-version }}.tar.gz\n ${{ matrix.edgedb-basename }}-${{ matrix.edgedb-version }}/bin/edgedb-server -D test-dir --bootstrap-only --testmode\n\n - name: Create databases on the older version\n if: ${{ matrix.make-dbs }}\n run: python3 .github/scripts/patches/create-databases.py ${{ matrix.edgedb-basename }}-${{ matrix.edgedb-version }}/bin/edgedb-server\n\n - name: Run tests with instance created on an older version\n run: |\n # Run the server explicitly first to do the upgrade, since edb test\n # has timeouts.\n edb server --bootstrap-only --data-dir test-dir\n # Should we run *all* the tests?\n edb test -j2 -v --data-dir test-dir tests/test_edgeql_json.py tests/test_edgeql_casts.py tests/test_edgeql_functions.py tests/test_edgeql_expressions.py tests/test_edgeql_policies.py tests/test_edgeql_vector.py tests/test_edgeql_scope.py tests/test_http_ext_auth.py\n\n - name: Test downgrading a database after an upgrade\n if: ${{ !contains(matrix.edgedb-version, '-rc') && !contains(matrix.edgedb-version, '-beta') }}\n env:\n EDGEDB_VERSION: ${{ matrix.edgedb-version }}\n run: python3 .github/scripts/patches/test-downgrade.py\n\n workflow-notifications:\n if: failure() && github.event_name != 'pull_request'\n name: Notify in Slack on failures\n needs:\n - build\n - compute-versions\n - test\n runs-on: ubuntu-latest\n permissions:\n actions: 'read'\n steps:\n - name: Slack Workflow Notification\n uses: Gamesight/slack-workflow-status@26a36836c887f260477432e4314ec3490a84f309\n with:\n repo_token: ${{secrets.GITHUB_TOKEN}}\n slack_webhook_url: ${{secrets.ACTIONS_SLACK_WEBHOOK_URL}}\n name: 'Workflow notifications'\n icon_emoji: ':hammer:'\n include_jobs: 'on-failure'\n" }, { "path": ".github/workflows/tests.pg-versions.yml", "content": "name: Tests on PostgreSQL Versions\n\non:\n schedule:\n - cron: \"0 3 * * *\"\n workflow_dispatch:\n inputs: {}\n push:\n branches:\n - pg-test\n\njobs:\n build:\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - uses: actions/checkout@v4\n with:\n fetch-depth: 50\n submodules: true\n\n - name: Set up Python\n uses: actions/setup-python@v5\n id: setup-python\n with:\n python-version: '3.12.2'\n cache: 'pip'\n cache-dependency-path: |\n pyproject.toml\n\n # The below is technically a lie as we are technically not\n # inside a virtual env, but there is really no reason to bother\n # actually creating and activating one as below works just fine.\n - name: Export $VIRTUAL_ENV\n run: |\n venv=\"$(python -c 'import sys; sys.stdout.write(sys.prefix)')\"\n echo \"VIRTUAL_ENV=${venv}\" >> $GITHUB_ENV\n\n - name: Set up uv cache\n uses: actions/cache@v4\n with:\n path: ~/.cache/uv\n key: uv-cache-${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}\n\n - name: Cached requirements.txt\n uses: actions/cache@v4\n id: requirements-cache\n with:\n path: requirements.txt\n key: edb-requirements-${{ hashFiles('pyproject.toml') }}\n\n - name: Compute requirements.txt\n if: steps.requirements-cache.outputs.cache-hit != 'true'\n run: |\n python -m pip install pip-tools\n pip-compile --no-strip-extras --all-build-deps \\\n --extra test,language-server \\\n --output-file requirements.txt pyproject.toml\n\n - name: Install Python dependencies\n run: |\n python -c \"import sys; print(sys.prefix)\"\n python -m pip install uv~=0.1.0 && uv pip install -U -r requirements.txt\n # 80.9.0 breaks our sphinx, and it keeps sneaking in\n uv pip install setuptools==80.8.0\n\n - name: Compute cache keys\n run: |\n mkdir -p shared-artifacts\n if [ \"$(uname)\" = \"Darwin\" ]; then\n find /usr/lib -type f -name 'lib*' -exec stat -f '%N %z' {} + | sort | shasum -a 256 | cut -d ' ' -f1 > shared-artifacts/lib_cache_key.txt\n else\n find /usr/lib -type f -name 'lib*' -printf '%P %s\\n' | sort | sha256sum | cut -d ' ' -f1 > shared-artifacts/lib_cache_key.txt\n fi\n python setup.py -q ci_helper --type rust >shared-artifacts/rust_cache_key.txt\n python setup.py -q ci_helper --type ext >shared-artifacts/ext_cache_key.txt\n python setup.py -q ci_helper --type parsers >shared-artifacts/parsers_cache_key.txt\n python setup.py -q ci_helper --type postgres >shared-artifacts/postgres_git_rev.txt\n python setup.py -q ci_helper --type libpg_query >shared-artifacts/libpg_query_git_rev.txt\n echo 'f8cd94309eaccbfba5dea7835b88c78377608a37' >shared-artifacts/stolon_git_rev.txt\n python setup.py -q ci_helper --type bootstrap >shared-artifacts/bootstrap_cache_key.txt\n echo POSTGRES_GIT_REV=$(cat shared-artifacts/postgres_git_rev.txt) >> $GITHUB_ENV\n echo LIBPG_QUERY_GIT_REV=$(cat shared-artifacts/libpg_query_git_rev.txt) >> $GITHUB_ENV\n echo STOLON_GIT_REV=$(cat shared-artifacts/stolon_git_rev.txt) >> $GITHUB_ENV\n echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV\n echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV\n\n - name: Upload shared artifacts\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: shared-artifacts\n path: shared-artifacts\n retention-days: 1\n\n # Restore binary cache\n\n - name: Handle cached Rust extensions\n uses: actions/cache@v4\n id: rust-cache\n with:\n path: build/rust_extensions\n key: edb-rust-v4-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n restore-keys: |\n edb-rust-v4-\n\n - name: Handle cached Cython extensions\n uses: actions/cache@v4\n id: ext-cache\n with:\n path: build/extensions\n key: edb-ext-v6-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Handle cached PostgreSQL build\n uses: actions/cache@v4\n id: postgres-cache\n with:\n path: build/postgres/install\n key: edb-postgres-v3-${{ env.POSTGRES_GIT_REV }}-${{ hashFiles('shared-artifacts/lib_cache_key.txt') }}\n\n - name: Handle cached Stolon build\n uses: actions/cache@v4\n id: stolon-cache\n with:\n path: build/stolon/bin\n key: edb-stolon-v2-${{ env.STOLON_GIT_REV }}\n\n - name: Handle cached libpg_query build\n uses: actions/cache@v4\n id: libpg-query-cache\n with:\n path: edb/pgsql/parser/libpg_query/libpg_query.a\n key: edb-libpg_query-v1-${{ env.LIBPG_QUERY_GIT_REV }}\n\n # Install system dependencies for building\n\n - name: Install system deps\n if: |\n steps.rust-cache.outputs.cache-hit != 'true' ||\n steps.ext-cache.outputs.cache-hit != 'true' ||\n steps.stolon-cache.outputs.cache-hit != 'true' ||\n steps.postgres-cache.outputs.cache-hit != 'true'\n run: |\n sudo apt-get update\n sudo apt-get install -y uuid-dev libreadline-dev bison flex libprotobuf-c-dev\n\n - name: Install Rust toolchain\n if: steps.rust-cache.outputs.cache-hit != 'true'\n uses: dsherret/rust-toolchain-file@v1\n\n # Build Rust extensions\n\n - name: Handle Rust extensions build cache\n uses: actions/cache@v4\n if: steps.rust-cache.outputs.cache-hit != 'true'\n with:\n path: ${{ env.BUILD_TEMP }}/rust/extensions\n key: edb-rust-build-v1-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n restore-keys: |\n edb-rust-build-v1-\n\n - name: Build Rust extensions\n env:\n CARGO_HOME: ${{ env.BUILD_TEMP }}/rust/extensions/cargo_home\n CACHE_HIT: ${{ steps.rust-cache.outputs.cache-hit }}\n run: |\n if [[ \"$CACHE_HIT\" != \"true\" ]]; then\n rm -rf ${BUILD_LIB}\n mkdir -p build/rust_extensions\n rsync -av ./build/rust_extensions/ ${BUILD_LIB}/\n python setup.py -v build_rust\n rsync -av ${BUILD_LIB}/ build/rust_extensions/\n rm -rf ${BUILD_LIB}\n fi\n rsync -av ./build/rust_extensions/edb/ ./edb/\n\n # Build libpg_query\n\n - name: Build libpg_query\n if: |\n steps.libpg-query-cache.outputs.cache-hit != 'true' &&\n steps.ext-cache.outputs.cache-hit != 'true'\n run: |\n python setup.py build_libpg_query\n\n # Build extensions\n\n - name: Handle Cython extensions build cache\n uses: actions/cache@v4\n if: steps.ext-cache.outputs.cache-hit != 'true'\n with:\n path: ${{ env.BUILD_TEMP }}/edb\n key: edb-ext-build-v4-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Build Cython extensions\n env:\n CACHE_HIT: ${{ steps.ext-cache.outputs.cache-hit }}\n BUILD_EXT_MODE: py-only\n run: |\n if [[ \"$CACHE_HIT\" != \"true\" ]]; then\n rm -rf ${BUILD_LIB}\n mkdir -p ./build/extensions\n rsync -av ./build/extensions/ ${BUILD_LIB}/\n BUILD_EXT_MODE=py-only python setup.py -v build_ext\n rsync -av ${BUILD_LIB}/ ./build/extensions/\n rm -rf ${BUILD_LIB}\n fi\n rsync -av ./build/extensions/edb/ ./edb/\n\n # Build parsers\n\n - name: Handle compiled parsers cache\n uses: actions/cache@v4\n id: parsers-cache\n with:\n path: build/lib\n key: edb-parsers-v3-${{ hashFiles('shared-artifacts/parsers_cache_key.txt') }}\n restore-keys: |\n edb-parsers-v3-\n\n - name: Build parsers\n env:\n CACHE_HIT: ${{ steps.parsers-cache.outputs.cache-hit }}\n run: |\n if [[ \"$CACHE_HIT\" != \"true\" ]]; then\n rm -rf ${BUILD_LIB}\n mkdir -p ./build/lib\n rsync -av ./build/lib/ ${BUILD_LIB}/\n python setup.py -v build_parsers\n rsync -av ${BUILD_LIB}/ ./build/lib/\n rm -rf ${BUILD_LIB}\n fi\n rsync -av ./build/lib/edb/ ./edb/\n\n # Build PostgreSQL\n\n - name: Build PostgreSQL\n env:\n CACHE_HIT: ${{ steps.postgres-cache.outputs.cache-hit }}\n run: |\n if [[ \"$CACHE_HIT\" == \"true\" ]]; then\n cp build/postgres/install/stamp build/postgres/\n else\n python setup.py build_postgres\n cp build/postgres/stamp build/postgres/install/\n fi\n\n # Build Stolon\n\n - name: Set up Go\n if: steps.stolon-cache.outputs.cache-hit != 'true'\n uses: actions/setup-go@v2\n with:\n go-version: 1.16\n\n - uses: actions/checkout@v4\n if: steps.stolon-cache.outputs.cache-hit != 'true'\n with:\n repository: edgedb/stolon\n path: build/stolon\n ref: ${{ env.STOLON_GIT_REV }}\n fetch-depth: 0\n submodules: false\n\n - name: Build Stolon\n if: steps.stolon-cache.outputs.cache-hit != 'true'\n run: |\n mkdir -p build/stolon/bin/\n curl -fsSL https://releases.hashicorp.com/consul/1.10.1/consul_1.10.1_linux_amd64.zip | zcat > build/stolon/bin/consul\n chmod +x build/stolon/bin/consul\n cd build/stolon && make\n\n # Install edgedb-server and populate egg-info\n\n - name: Install edgedb-server\n env:\n BUILD_EXT_MODE: skip\n run: |\n # --no-build-isolation because we have explicitly installed all deps\n # and don't want them to be reinstalled in an \"isolated env\".\n pip install --no-build-isolation --no-deps -e .[test,docs]\n\n # Refresh the bootstrap cache\n\n - name: Handle bootstrap cache\n uses: actions/cache@v4\n id: bootstrap-cache\n with:\n path: build/cache\n key: edb-bootstrap-v2-${{ hashFiles('shared-artifacts/bootstrap_cache_key.txt') }}\n restore-keys: |\n edb-bootstrap-v2-\n\n - name: Bootstrap EdgeDB Server\n if: steps.bootstrap-cache.outputs.cache-hit != 'true'\n run: |\n edb server --bootstrap-only\n\n test:\n runs-on: ubuntu-latest\n needs: build\n strategy:\n fail-fast: false\n matrix:\n postgres-version: [ 17 ]\n single-mode:\n - ''\n # These are very broken. Disabling them for now until we\n # decide whether to fix them or give up.\n # - 'NOCREATEDB NOCREATEROLE'\n # - 'CREATEDB NOCREATEROLE'\n multi-tenant-mode: [ '' ]\n include:\n - postgres-version: 14\n single-mode: ''\n multi-tenant-mode: ''\n - postgres-version: 15\n single-mode: ''\n multi-tenant-mode: ''\n - postgres-version: 16\n single-mode: ''\n multi-tenant-mode: ''\n - postgres-version: 17\n single-mode: ''\n multi-tenant-mode: 'remote-compiler'\n - postgres-version: 17\n single-mode: ''\n multi-tenant-mode: 'multi-tenant'\n services:\n postgres:\n image: pgvector/pgvector:0.7.4-pg${{ matrix.postgres-version }}\n env:\n POSTGRES_PASSWORD: postgres\n options: >-\n --health-cmd pg_isready\n --health-interval 10s\n --health-timeout 5s\n --health-retries 5\n --name postgres\n ports:\n - 5432:5432\n\n steps:\n - name: Trust pgvector extension\n uses: docker://docker\n with:\n args: docker exec postgres sed -i $a\\trusted=true /usr/share/postgresql/${{ matrix.postgres-version }}/extension/vector.control\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - uses: actions/checkout@v4\n with:\n fetch-depth: 50\n submodules: true\n\n - name: Set up Python\n uses: actions/setup-python@v5\n id: setup-python\n with:\n python-version: '3.12.2'\n cache: 'pip'\n cache-dependency-path: |\n pyproject.toml\n\n # The below is technically a lie as we are technically not\n # inside a virtual env, but there is really no reason to bother\n # actually creating and activating one as below works just fine.\n - name: Export $VIRTUAL_ENV\n run: |\n venv=\"$(python -c 'import sys; sys.stdout.write(sys.prefix)')\"\n echo \"VIRTUAL_ENV=${venv}\" >> $GITHUB_ENV\n\n - name: Set up uv cache\n uses: actions/cache@v4\n with:\n path: ~/.cache/uv\n key: uv-cache-${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}\n \n - name: Download requirements.txt\n uses: actions/cache@v4\n with:\n path: requirements.txt\n key: edb-requirements-${{ hashFiles('pyproject.toml') }}\n\n - name: Install Python dependencies\n run: |\n python -m pip install uv~=0.1.0 && uv pip install -U -r requirements.txt\n # 80.9.0 breaks our sphinx, and it keeps sneaking in\n uv pip install setuptools==80.8.0\n\n # Restore the artifacts and environment variables\n\n - name: Download shared artifacts\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: shared-artifacts\n path: shared-artifacts\n\n - name: Set environment variables\n run: |\n echo POSTGRES_GIT_REV=$(cat shared-artifacts/postgres_git_rev.txt) >> $GITHUB_ENV\n echo STOLON_GIT_REV=$(cat shared-artifacts/stolon_git_rev.txt) >> $GITHUB_ENV\n echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV\n echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV\n\n # Restore build cache\n\n - name: Restore cached Rust extensions\n uses: actions/cache@v4\n id: rust-cache\n with:\n path: build/rust_extensions\n key: edb-rust-v4-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n\n - name: Restore cached Cython extensions\n uses: actions/cache@v4\n id: ext-cache\n with:\n path: build/extensions\n key: edb-ext-v6-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Restore compiled parsers cache\n uses: actions/cache@v4\n id: parsers-cache\n with:\n path: build/lib\n key: edb-parsers-v3-${{ hashFiles('shared-artifacts/parsers_cache_key.txt') }}\n\n - name: Restore cached PostgreSQL build\n uses: actions/cache@v4\n id: postgres-cache\n with:\n path: build/postgres/install\n key: edb-postgres-v3-${{ env.POSTGRES_GIT_REV }}-${{ hashFiles('shared-artifacts/lib_cache_key.txt') }}\n\n - name: Restore cached Stolon build\n uses: actions/cache@v4\n id: stolon-cache\n with:\n path: build/stolon/bin\n key: edb-stolon-v2-${{ env.STOLON_GIT_REV }}\n\n - name: Restore bootstrap cache\n uses: actions/cache@v4\n id: bootstrap-cache\n with:\n path: build/cache\n key: edb-bootstrap-v2-${{ hashFiles('shared-artifacts/bootstrap_cache_key.txt') }}\n\n - name: Stop if we cannot retrieve the cache\n if: |\n steps.rust-cache.outputs.cache-hit != 'true' ||\n steps.ext-cache.outputs.cache-hit != 'true' ||\n steps.parsers-cache.outputs.cache-hit != 'true' ||\n steps.postgres-cache.outputs.cache-hit != 'true' ||\n steps.stolon-cache.outputs.cache-hit != 'true' ||\n steps.bootstrap-cache.outputs.cache-hit != 'true'\n run: |\n echo ::error::Cannot retrieve build cache.\n exit 1\n\n - name: Validate cached binaries\n run: |\n # Validate Stolon\n ./build/stolon/bin/stolon-sentinel --version || exit 1\n ./build/stolon/bin/stolon-keeper --version || exit 1\n ./build/stolon/bin/stolon-proxy --version || exit 1\n\n # Validate PostgreSQL\n ./build/postgres/install/bin/postgres --version || exit 1\n ./build/postgres/install/bin/pg_config --version || exit 1\n\n - name: Restore cache into the source tree\n run: |\n rsync -av ./build/rust_extensions/edb/ ./edb/\n rsync -av ./build/extensions/edb/ ./edb/\n rsync -av ./build/lib/edb/ ./edb/\n cp build/postgres/install/stamp build/postgres/\n\n - name: Install edgedb-server\n env:\n BUILD_EXT_MODE: skip\n run: |\n # --no-build-isolation because we have explicitly installed all deps\n # and don't want them to be reinstalled in an \"isolated env\".\n pip install --no-build-isolation --no-deps -e .[test,docs]\n\n # Run the test\n\n - name: Setup single mode role and database\n if: ${{ matrix.single-mode }}\n shell: python\n run: |\n import asyncio\n import subprocess\n\n from edb.server.pgcluster import get_pg_bin_dir\n\n async def main():\n psql = await get_pg_bin_dir() / \"psql\"\n dsn = \"postgres://postgres:postgres@localhost/postgres\"\n\n script = \"\"\"\\\n CREATE ROLE singles;\n ALTER ROLE singles WITH LOGIN PASSWORD 'test' NOSUPERUSER\n ${{ matrix.single-mode }};\n CREATE DATABASE singles OWNER singles;\n REVOKE ALL ON DATABASE singles FROM PUBLIC;\n GRANT CONNECT ON DATABASE singles TO singles;\n GRANT ALL ON DATABASE singles TO singles;\n \"\"\"\n\n subprocess.run(\n [str(psql), dsn],\n check=True,\n text=True,\n input=script,\n )\n\n asyncio.run(main())\n\n - name: Test\n env:\n EDGEDB_TEST_POSTGRES_VERSION: ${{ matrix.postgres-version }}\n run: |\n if [[ \"${{ matrix.single-mode }}\" ]]; then\n export EDGEDB_TEST_BACKEND_DSN=postgres://singles:test@localhost/singles\n else\n export EDGEDB_TEST_BACKEND_DSN=postgres://postgres:postgres@localhost/postgres\n fi\n if [[ \"${{ matrix.multi-tenant-mode }}\" == \"remote-compiler\" ]]; then\n export EDGEDB_TEST_REMOTE_COMPILER=localhost:5660\n export _EDGEDB_SERVER_COMPILER_POOL_SECRET=secret\n __EDGEDB_DEVMODE=1 edgedb-server compiler --pool-size 2 &\n fi\n edb server --bootstrap-only --backend-dsn=$EDGEDB_TEST_BACKEND_DSN --testmode\n if [[ \"${{ matrix.multi-tenant-mode }}\" == \"multi-tenant\" ]]; then\n export EDGEDB_SERVER_MULTITENANT_CONFIG_FILE=/tmp/edb.mt.json\n echo \"{\\\"localhost\\\":{\\\"instance-name\\\":\\\"localtest\\\",\\\"backend-dsn\\\":\\\"$EDGEDB_TEST_BACKEND_DSN\\\",\\\"admin\\\":true,\\\"max-backend-connections\\\":10}}\" > /tmp/edb.mt.json\n fi\n if [[ \"${{ matrix.single-mode }}\" == *\"NOCREATEDB\"* ]]; then\n edb test -j1 -v --backend-dsn=$EDGEDB_TEST_BACKEND_DSN\n else\n edb test -j2 -v --backend-dsn=$EDGEDB_TEST_BACKEND_DSN\n fi\n\n\n workflow-notifications:\n if: failure() && github.event_name != 'pull_request'\n name: Notify in Slack on failures\n needs:\n - build\n - test\n runs-on: ubuntu-latest\n permissions:\n actions: 'read'\n steps:\n - name: Slack Workflow Notification\n uses: Gamesight/slack-workflow-status@26a36836c887f260477432e4314ec3490a84f309\n with:\n repo_token: ${{secrets.GITHUB_TOKEN}}\n slack_webhook_url: ${{secrets.ACTIONS_SLACK_WEBHOOK_URL}}\n name: 'Workflow notifications'\n icon_emoji: ':hammer:'\n include_jobs: 'on-failure'\n" }, { "path": ".github/workflows/tests.pool.yml", "content": "name: Pool Simulation Test\n\non:\n push:\n branches:\n - master\n - pool-test\n paths:\n - 'edb/server/connpool/**'\n - 'edb/server/conn_pool/**'\n - 'tests/test_server_pool.py'\n - '.github/workflows/tests-pool.yml'\n pull_request:\n branches:\n - master\n paths:\n - 'edb/server/connpool/**'\n - 'edb/server/conn_pool/**'\n - 'tests/test_server_pool.py'\n - '.github/workflows/tests-pool.yml'\n\njobs:\n test:\n runs-on: ubuntu-latest\n concurrency: pool-test\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - uses: actions/checkout@v4\n with:\n fetch-depth: 50\n submodules: true\n\n - name: Set up Python\n uses: actions/setup-python@v5\n id: setup-python\n with:\n python-version: '3.12.2'\n cache: 'pip'\n cache-dependency-path: |\n pyproject.toml\n\n # The below is technically a lie as we are technically not\n # inside a virtual env, but there is really no reason to bother\n # actually creating and activating one as below works just fine.\n - name: Export $VIRTUAL_ENV\n run: |\n venv=\"$(python -c 'import sys; sys.stdout.write(sys.prefix)')\"\n echo \"VIRTUAL_ENV=${venv}\" >> $GITHUB_ENV\n\n - name: Set up uv cache\n uses: actions/cache@v4\n with:\n path: ~/.cache/uv\n key: uv-cache-${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}\n\n - name: Cached requirements.txt\n uses: actions/cache@v4\n id: requirements-cache\n with:\n path: requirements.txt\n key: edb-requirements-${{ hashFiles('pyproject.toml') }}\n\n - name: Compute requirements.txt\n if: steps.requirements-cache.outputs.cache-hit != 'true'\n run: |\n python -m pip install pip-tools\n pip-compile --no-strip-extras --all-build-deps \\\n --extra test,language-server \\\n --output-file requirements.txt pyproject.toml\n\n - name: Install Python dependencies\n run: |\n python -c \"import sys; print(sys.prefix)\"\n python -m pip install uv~=0.1.0 && uv pip install -U -r requirements.txt\n # 80.9.0 breaks our sphinx, and it keeps sneaking in\n uv pip install setuptools==80.8.0\n\n - name: Compute cache keys\n run: |\n mkdir -p shared-artifacts\n if [ \"$(uname)\" = \"Darwin\" ]; then\n find /usr/lib -type f -name 'lib*' -exec stat -f '%N %z' {} + | sort | shasum -a 256 | cut -d ' ' -f1 > shared-artifacts/lib_cache_key.txt\n else\n find /usr/lib -type f -name 'lib*' -printf '%P %s\\n' | sort | sha256sum | cut -d ' ' -f1 > shared-artifacts/lib_cache_key.txt\n fi\n python setup.py -q ci_helper --type rust >shared-artifacts/rust_cache_key.txt\n python setup.py -q ci_helper --type ext >shared-artifacts/ext_cache_key.txt\n python setup.py -q ci_helper --type parsers >shared-artifacts/parsers_cache_key.txt\n python setup.py -q ci_helper --type postgres >shared-artifacts/postgres_git_rev.txt\n python setup.py -q ci_helper --type libpg_query >shared-artifacts/libpg_query_git_rev.txt\n echo 'f8cd94309eaccbfba5dea7835b88c78377608a37' >shared-artifacts/stolon_git_rev.txt\n python setup.py -q ci_helper --type bootstrap >shared-artifacts/bootstrap_cache_key.txt\n echo POSTGRES_GIT_REV=$(cat shared-artifacts/postgres_git_rev.txt) >> $GITHUB_ENV\n echo LIBPG_QUERY_GIT_REV=$(cat shared-artifacts/libpg_query_git_rev.txt) >> $GITHUB_ENV\n echo STOLON_GIT_REV=$(cat shared-artifacts/stolon_git_rev.txt) >> $GITHUB_ENV\n echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV\n echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV\n\n - name: Upload shared artifacts\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: shared-artifacts\n path: shared-artifacts\n retention-days: 1\n\n # Restore binary cache\n\n - name: Handle cached Rust extensions\n uses: actions/cache@v4\n id: rust-cache\n with:\n path: build/rust_extensions\n key: edb-rust-v4-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n restore-keys: |\n edb-rust-v4-\n\n - name: Handle cached Cython extensions\n uses: actions/cache@v4\n id: ext-cache\n with:\n path: build/extensions\n key: edb-ext-v6-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Handle cached PostgreSQL build\n uses: actions/cache@v4\n id: postgres-cache\n with:\n path: build/postgres/install\n key: edb-postgres-v3-${{ env.POSTGRES_GIT_REV }}-${{ hashFiles('shared-artifacts/lib_cache_key.txt') }}\n\n - name: Handle cached Stolon build\n uses: actions/cache@v4\n id: stolon-cache\n with:\n path: build/stolon/bin\n key: edb-stolon-v2-${{ env.STOLON_GIT_REV }}\n\n - name: Handle cached libpg_query build\n uses: actions/cache@v4\n id: libpg-query-cache\n with:\n path: edb/pgsql/parser/libpg_query/libpg_query.a\n key: edb-libpg_query-v1-${{ env.LIBPG_QUERY_GIT_REV }}\n\n # Install system dependencies for building\n\n - name: Install system deps\n if: |\n steps.rust-cache.outputs.cache-hit != 'true' ||\n steps.ext-cache.outputs.cache-hit != 'true' ||\n steps.stolon-cache.outputs.cache-hit != 'true' ||\n steps.postgres-cache.outputs.cache-hit != 'true'\n run: |\n sudo apt-get update\n sudo apt-get install -y uuid-dev libreadline-dev bison flex libprotobuf-c-dev\n\n - name: Install Rust toolchain\n if: steps.rust-cache.outputs.cache-hit != 'true'\n uses: dsherret/rust-toolchain-file@v1\n\n # Build Rust extensions\n\n - name: Handle Rust extensions build cache\n uses: actions/cache@v4\n if: steps.rust-cache.outputs.cache-hit != 'true'\n with:\n path: ${{ env.BUILD_TEMP }}/rust/extensions\n key: edb-rust-build-v1-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n restore-keys: |\n edb-rust-build-v1-\n\n - name: Build Rust extensions\n env:\n CARGO_HOME: ${{ env.BUILD_TEMP }}/rust/extensions/cargo_home\n CACHE_HIT: ${{ steps.rust-cache.outputs.cache-hit }}\n run: |\n if [[ \"$CACHE_HIT\" != \"true\" ]]; then\n rm -rf ${BUILD_LIB}\n mkdir -p build/rust_extensions\n rsync -av ./build/rust_extensions/ ${BUILD_LIB}/\n python setup.py -v build_rust\n rsync -av ${BUILD_LIB}/ build/rust_extensions/\n rm -rf ${BUILD_LIB}\n fi\n rsync -av ./build/rust_extensions/edb/ ./edb/\n\n # Build libpg_query\n\n - name: Build libpg_query\n if: |\n steps.libpg-query-cache.outputs.cache-hit != 'true' &&\n steps.ext-cache.outputs.cache-hit != 'true'\n run: |\n python setup.py build_libpg_query\n\n # Build extensions\n\n - name: Handle Cython extensions build cache\n uses: actions/cache@v4\n if: steps.ext-cache.outputs.cache-hit != 'true'\n with:\n path: ${{ env.BUILD_TEMP }}/edb\n key: edb-ext-build-v4-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Build Cython extensions\n env:\n CACHE_HIT: ${{ steps.ext-cache.outputs.cache-hit }}\n BUILD_EXT_MODE: py-only\n run: |\n if [[ \"$CACHE_HIT\" != \"true\" ]]; then\n rm -rf ${BUILD_LIB}\n mkdir -p ./build/extensions\n rsync -av ./build/extensions/ ${BUILD_LIB}/\n BUILD_EXT_MODE=py-only python setup.py -v build_ext\n rsync -av ${BUILD_LIB}/ ./build/extensions/\n rm -rf ${BUILD_LIB}\n fi\n rsync -av ./build/extensions/edb/ ./edb/\n\n # Build parsers\n\n - name: Handle compiled parsers cache\n uses: actions/cache@v4\n id: parsers-cache\n with:\n path: build/lib\n key: edb-parsers-v3-${{ hashFiles('shared-artifacts/parsers_cache_key.txt') }}\n restore-keys: |\n edb-parsers-v3-\n\n - name: Build parsers\n env:\n CACHE_HIT: ${{ steps.parsers-cache.outputs.cache-hit }}\n run: |\n if [[ \"$CACHE_HIT\" != \"true\" ]]; then\n rm -rf ${BUILD_LIB}\n mkdir -p ./build/lib\n rsync -av ./build/lib/ ${BUILD_LIB}/\n python setup.py -v build_parsers\n rsync -av ${BUILD_LIB}/ ./build/lib/\n rm -rf ${BUILD_LIB}\n fi\n rsync -av ./build/lib/edb/ ./edb/\n\n # Build PostgreSQL\n\n - name: Build PostgreSQL\n env:\n CACHE_HIT: ${{ steps.postgres-cache.outputs.cache-hit }}\n run: |\n if [[ \"$CACHE_HIT\" == \"true\" ]]; then\n cp build/postgres/install/stamp build/postgres/\n else\n python setup.py build_postgres\n cp build/postgres/stamp build/postgres/install/\n fi\n\n # Build Stolon\n\n - name: Set up Go\n if: steps.stolon-cache.outputs.cache-hit != 'true'\n uses: actions/setup-go@v2\n with:\n go-version: 1.16\n\n - uses: actions/checkout@v4\n if: steps.stolon-cache.outputs.cache-hit != 'true'\n with:\n repository: edgedb/stolon\n path: build/stolon\n ref: ${{ env.STOLON_GIT_REV }}\n fetch-depth: 0\n submodules: false\n\n - name: Build Stolon\n if: steps.stolon-cache.outputs.cache-hit != 'true'\n run: |\n mkdir -p build/stolon/bin/\n curl -fsSL https://releases.hashicorp.com/consul/1.10.1/consul_1.10.1_linux_amd64.zip | zcat > build/stolon/bin/consul\n chmod +x build/stolon/bin/consul\n cd build/stolon && make\n\n # Install edgedb-server and populate egg-info\n\n - name: Install edgedb-server\n env:\n BUILD_EXT_MODE: skip\n run: |\n # --no-build-isolation because we have explicitly installed all deps\n # and don't want them to be reinstalled in an \"isolated env\".\n pip install --no-build-isolation --no-deps -e .[test,docs]\n\n # Refresh the bootstrap cache\n\n - name: Handle bootstrap cache\n uses: actions/cache@v4\n id: bootstrap-cache\n with:\n path: build/cache\n key: edb-bootstrap-v2-${{ hashFiles('shared-artifacts/bootstrap_cache_key.txt') }}\n restore-keys: |\n edb-bootstrap-v2-\n\n - name: Bootstrap EdgeDB Server\n if: steps.bootstrap-cache.outputs.cache-hit != 'true'\n run: |\n edb server --bootstrap-only\n\n - uses: actions/checkout@v4\n if: startsWith(github.ref, 'refs/heads')\n with:\n repository: edgedb/edgedb-pool-simulation\n path: pool-simulation\n token: ${{ secrets.GITHUB_CI_BOT_TOKEN }}\n\n - name: Run the pool simulation test\n env:\n PYTHONPATH: .\n SIMULATION_CI: yes\n TIME_SCALE: 10\n run: |\n mkdir -p pool-simulation/reports\n python tests/test_server_pool.py\n\n - uses: EndBug/add-and-commit@v7.0.0\n if: ${{ always() }}\n continue-on-error: true\n with:\n branch: main\n cwd: pool-simulation\n author_name: github-actions\n author_email: 41898282+github-actions[bot]@users.noreply.github.com\n" }, { "path": ".github/workflows/tests.reflection.yml", "content": "name: Tests with reflection validation\n\non:\n schedule:\n - cron: \"0 3 * * *\"\n workflow_dispatch:\n inputs: {}\n push:\n branches:\n - \"REFL-*\"\n\njobs:\n build:\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - uses: actions/checkout@v4\n with:\n fetch-depth: 50\n submodules: true\n\n - name: Set up Python\n uses: actions/setup-python@v5\n id: setup-python\n with:\n python-version: '3.12.2'\n cache: 'pip'\n cache-dependency-path: |\n pyproject.toml\n\n # The below is technically a lie as we are technically not\n # inside a virtual env, but there is really no reason to bother\n # actually creating and activating one as below works just fine.\n - name: Export $VIRTUAL_ENV\n run: |\n venv=\"$(python -c 'import sys; sys.stdout.write(sys.prefix)')\"\n echo \"VIRTUAL_ENV=${venv}\" >> $GITHUB_ENV\n\n - name: Set up uv cache\n uses: actions/cache@v4\n with:\n path: ~/.cache/uv\n key: uv-cache-${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}\n\n - name: Cached requirements.txt\n uses: actions/cache@v4\n id: requirements-cache\n with:\n path: requirements.txt\n key: edb-requirements-${{ hashFiles('pyproject.toml') }}\n\n - name: Compute requirements.txt\n if: steps.requirements-cache.outputs.cache-hit != 'true'\n run: |\n python -m pip install pip-tools\n pip-compile --no-strip-extras --all-build-deps \\\n --extra test,language-server \\\n --output-file requirements.txt pyproject.toml\n\n - name: Install Python dependencies\n run: |\n python -c \"import sys; print(sys.prefix)\"\n python -m pip install uv~=0.1.0 && uv pip install -U -r requirements.txt\n # 80.9.0 breaks our sphinx, and it keeps sneaking in\n uv pip install setuptools==80.8.0\n\n - name: Compute cache keys\n env:\n GIST_TOKEN: ${{ secrets.CI_BOT_GIST_TOKEN }}\n run: |\n mkdir -p shared-artifacts\n if [ \"$(uname)\" = \"Darwin\" ]; then\n find /usr/lib -type f -name 'lib*' -exec stat -f '%N %z' {} + | sort | shasum -a 256 | cut -d ' ' -f1 > shared-artifacts/lib_cache_key.txt\n else\n find /usr/lib -type f -name 'lib*' -printf '%P %s\\n' | sort | sha256sum | cut -d ' ' -f1 > shared-artifacts/lib_cache_key.txt\n fi\n python setup.py -q ci_helper --type rust >shared-artifacts/rust_cache_key.txt\n python setup.py -q ci_helper --type ext >shared-artifacts/ext_cache_key.txt\n python setup.py -q ci_helper --type parsers >shared-artifacts/parsers_cache_key.txt\n python setup.py -q ci_helper --type postgres >shared-artifacts/postgres_git_rev.txt\n python setup.py -q ci_helper --type libpg_query >shared-artifacts/libpg_query_git_rev.txt\n echo 'f8cd94309eaccbfba5dea7835b88c78377608a37' >shared-artifacts/stolon_git_rev.txt\n python setup.py -q ci_helper --type bootstrap >shared-artifacts/bootstrap_cache_key.txt\n echo POSTGRES_GIT_REV=$(cat shared-artifacts/postgres_git_rev.txt) >> $GITHUB_ENV\n echo LIBPG_QUERY_GIT_REV=$(cat shared-artifacts/libpg_query_git_rev.txt) >> $GITHUB_ENV\n echo STOLON_GIT_REV=$(cat shared-artifacts/stolon_git_rev.txt) >> $GITHUB_ENV\n echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV\n echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV\n\n - name: Upload shared artifacts\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: shared-artifacts\n path: shared-artifacts\n retention-days: 1\n\n # Restore binary cache\n\n - name: Handle cached Rust extensions\n uses: actions/cache@v4\n id: rust-cache\n with:\n path: build/rust_extensions\n key: edb-rust-v4-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n restore-keys: |\n edb-rust-v4-\n\n - name: Handle cached Cython extensions\n uses: actions/cache@v4\n id: ext-cache\n with:\n path: build/extensions\n key: edb-ext-v6-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Handle cached PostgreSQL build\n uses: actions/cache@v4\n id: postgres-cache\n with:\n path: build/postgres/install\n key: edb-postgres-v3-${{ env.POSTGRES_GIT_REV }}-${{ hashFiles('shared-artifacts/lib_cache_key.txt') }}\n\n - name: Handle cached Stolon build\n uses: actions/cache@v4\n id: stolon-cache\n with:\n path: build/stolon/bin\n key: edb-stolon-v2-${{ env.STOLON_GIT_REV }}\n\n - name: Handle cached libpg_query build\n uses: actions/cache@v4\n id: libpg-query-cache\n with:\n path: edb/pgsql/parser/libpg_query/libpg_query.a\n key: edb-libpg_query-v1-${{ env.LIBPG_QUERY_GIT_REV }}\n\n # Install system dependencies for building\n\n - name: Install system deps\n if: |\n steps.rust-cache.outputs.cache-hit != 'true' ||\n steps.ext-cache.outputs.cache-hit != 'true' ||\n steps.stolon-cache.outputs.cache-hit != 'true' ||\n steps.postgres-cache.outputs.cache-hit != 'true'\n run: |\n sudo apt-get update\n sudo apt-get install -y uuid-dev libreadline-dev bison flex libprotobuf-c-dev\n\n - name: Install Rust toolchain\n if: steps.rust-cache.outputs.cache-hit != 'true'\n uses: dsherret/rust-toolchain-file@v1\n\n # Build Rust extensions\n\n - name: Handle Rust extensions build cache\n uses: actions/cache@v4\n if: steps.rust-cache.outputs.cache-hit != 'true'\n with:\n path: ${{ env.BUILD_TEMP }}/rust/extensions\n key: edb-rust-build-v1-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n restore-keys: |\n edb-rust-build-v1-\n\n - name: Build Rust extensions\n env:\n CARGO_HOME: ${{ env.BUILD_TEMP }}/rust/extensions/cargo_home\n CACHE_HIT: ${{ steps.rust-cache.outputs.cache-hit }}\n run: |\n if [[ \"$CACHE_HIT\" != \"true\" ]]; then\n rm -rf ${BUILD_LIB}\n mkdir -p build/rust_extensions\n rsync -av ./build/rust_extensions/ ${BUILD_LIB}/\n python setup.py -v build_rust\n rsync -av ${BUILD_LIB}/ build/rust_extensions/\n rm -rf ${BUILD_LIB}\n fi\n rsync -av ./build/rust_extensions/edb/ ./edb/\n\n # Build libpg_query\n\n - name: Build libpg_query\n if: |\n steps.libpg-query-cache.outputs.cache-hit != 'true' &&\n steps.ext-cache.outputs.cache-hit != 'true'\n run: |\n python setup.py build_libpg_query\n\n # Build extensions\n\n - name: Handle Cython extensions build cache\n uses: actions/cache@v4\n if: steps.ext-cache.outputs.cache-hit != 'true'\n with:\n path: ${{ env.BUILD_TEMP }}/edb\n key: edb-ext-build-v4-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Build Cython extensions\n env:\n CACHE_HIT: ${{ steps.ext-cache.outputs.cache-hit }}\n BUILD_EXT_MODE: py-only\n run: |\n if [[ \"$CACHE_HIT\" != \"true\" ]]; then\n rm -rf ${BUILD_LIB}\n mkdir -p ./build/extensions\n rsync -av ./build/extensions/ ${BUILD_LIB}/\n BUILD_EXT_MODE=py-only python setup.py -v build_ext\n rsync -av ${BUILD_LIB}/ ./build/extensions/\n rm -rf ${BUILD_LIB}\n fi\n rsync -av ./build/extensions/edb/ ./edb/\n\n # Build parsers\n\n - name: Handle compiled parsers cache\n uses: actions/cache@v4\n id: parsers-cache\n with:\n path: build/lib\n key: edb-parsers-v3-${{ hashFiles('shared-artifacts/parsers_cache_key.txt') }}\n restore-keys: |\n edb-parsers-v3-\n\n - name: Build parsers\n env:\n CACHE_HIT: ${{ steps.parsers-cache.outputs.cache-hit }}\n run: |\n if [[ \"$CACHE_HIT\" != \"true\" ]]; then\n rm -rf ${BUILD_LIB}\n mkdir -p ./build/lib\n rsync -av ./build/lib/ ${BUILD_LIB}/\n python setup.py -v build_parsers\n rsync -av ${BUILD_LIB}/ ./build/lib/\n rm -rf ${BUILD_LIB}\n fi\n rsync -av ./build/lib/edb/ ./edb/\n\n # Build PostgreSQL\n\n - name: Build PostgreSQL\n env:\n CACHE_HIT: ${{ steps.postgres-cache.outputs.cache-hit }}\n run: |\n if [[ \"$CACHE_HIT\" == \"true\" ]]; then\n cp build/postgres/install/stamp build/postgres/\n else\n python setup.py build_postgres\n cp build/postgres/stamp build/postgres/install/\n fi\n\n # Build Stolon\n\n - name: Set up Go\n if: steps.stolon-cache.outputs.cache-hit != 'true'\n uses: actions/setup-go@v2\n with:\n go-version: 1.16\n\n - uses: actions/checkout@v4\n if: steps.stolon-cache.outputs.cache-hit != 'true'\n with:\n repository: edgedb/stolon\n path: build/stolon\n ref: ${{ env.STOLON_GIT_REV }}\n fetch-depth: 0\n submodules: false\n\n - name: Build Stolon\n if: steps.stolon-cache.outputs.cache-hit != 'true'\n run: |\n mkdir -p build/stolon/bin/\n curl -fsSL https://releases.hashicorp.com/consul/1.10.1/consul_1.10.1_linux_amd64.zip | zcat > build/stolon/bin/consul\n chmod +x build/stolon/bin/consul\n cd build/stolon && make\n\n # Install edgedb-server and populate egg-info\n\n - name: Install edgedb-server\n env:\n BUILD_EXT_MODE: skip\n run: |\n # --no-build-isolation because we have explicitly installed all deps\n # and don't want them to be reinstalled in an \"isolated env\".\n pip install --no-build-isolation --no-deps -e .[test,docs]\n\n # Refresh the bootstrap cache\n\n - name: Handle bootstrap cache\n uses: actions/cache@v4\n id: bootstrap-cache\n with:\n path: build/cache\n key: edb-bootstrap-v2-${{ hashFiles('shared-artifacts/bootstrap_cache_key.txt') }}\n restore-keys: |\n edb-bootstrap-v2-\n\n - name: Bootstrap EdgeDB Server\n if: steps.bootstrap-cache.outputs.cache-hit != 'true'\n run: |\n edb server --bootstrap-only\n\n test:\n needs: build\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - uses: actions/checkout@v4\n with:\n fetch-depth: 50\n submodules: true\n\n - name: Set up Python\n uses: actions/setup-python@v5\n id: setup-python\n with:\n python-version: '3.12.2'\n cache: 'pip'\n cache-dependency-path: |\n pyproject.toml\n\n # The below is technically a lie as we are technically not\n # inside a virtual env, but there is really no reason to bother\n # actually creating and activating one as below works just fine.\n - name: Export $VIRTUAL_ENV\n run: |\n venv=\"$(python -c 'import sys; sys.stdout.write(sys.prefix)')\"\n echo \"VIRTUAL_ENV=${venv}\" >> $GITHUB_ENV\n\n - name: Set up uv cache\n uses: actions/cache@v4\n with:\n path: ~/.cache/uv\n key: uv-cache-${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}\n \n - name: Download requirements.txt\n uses: actions/cache@v4\n with:\n path: requirements.txt\n key: edb-requirements-${{ hashFiles('pyproject.toml') }}\n\n - name: Install Python dependencies\n run: |\n python -m pip install uv~=0.1.0 && uv pip install -U -r requirements.txt\n # 80.9.0 breaks our sphinx, and it keeps sneaking in\n uv pip install setuptools==80.8.0\n\n # Restore the artifacts and environment variables\n\n - name: Download shared artifacts\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: shared-artifacts\n path: shared-artifacts\n\n - name: Set environment variables\n run: |\n echo POSTGRES_GIT_REV=$(cat shared-artifacts/postgres_git_rev.txt) >> $GITHUB_ENV\n echo STOLON_GIT_REV=$(cat shared-artifacts/stolon_git_rev.txt) >> $GITHUB_ENV\n echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV\n echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV\n\n # Restore build cache\n\n - name: Restore cached Rust extensions\n uses: actions/cache@v4\n id: rust-cache\n with:\n path: build/rust_extensions\n key: edb-rust-v4-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n\n - name: Restore cached Cython extensions\n uses: actions/cache@v4\n id: ext-cache\n with:\n path: build/extensions\n key: edb-ext-v6-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Restore compiled parsers cache\n uses: actions/cache@v4\n id: parsers-cache\n with:\n path: build/lib\n key: edb-parsers-v3-${{ hashFiles('shared-artifacts/parsers_cache_key.txt') }}\n\n - name: Restore cached PostgreSQL build\n uses: actions/cache@v4\n id: postgres-cache\n with:\n path: build/postgres/install\n key: edb-postgres-v3-${{ env.POSTGRES_GIT_REV }}-${{ hashFiles('shared-artifacts/lib_cache_key.txt') }}\n\n - name: Restore cached Stolon build\n uses: actions/cache@v4\n id: stolon-cache\n with:\n path: build/stolon/bin\n key: edb-stolon-v2-${{ env.STOLON_GIT_REV }}\n\n - name: Restore bootstrap cache\n uses: actions/cache@v4\n id: bootstrap-cache\n with:\n path: build/cache\n key: edb-bootstrap-v2-${{ hashFiles('shared-artifacts/bootstrap_cache_key.txt') }}\n\n - name: Stop if we cannot retrieve the cache\n if: |\n steps.rust-cache.outputs.cache-hit != 'true' ||\n steps.ext-cache.outputs.cache-hit != 'true' ||\n steps.parsers-cache.outputs.cache-hit != 'true' ||\n steps.postgres-cache.outputs.cache-hit != 'true' ||\n steps.stolon-cache.outputs.cache-hit != 'true' ||\n steps.bootstrap-cache.outputs.cache-hit != 'true'\n run: |\n echo ::error::Cannot retrieve build cache.\n exit 1\n\n - name: Validate cached binaries\n run: |\n # Validate Stolon\n ./build/stolon/bin/stolon-sentinel --version || exit 1\n ./build/stolon/bin/stolon-keeper --version || exit 1\n ./build/stolon/bin/stolon-proxy --version || exit 1\n\n # Validate PostgreSQL\n ./build/postgres/install/bin/postgres --version || exit 1\n ./build/postgres/install/bin/pg_config --version || exit 1\n\n - name: Restore cache into the source tree\n run: |\n rsync -av ./build/rust_extensions/edb/ ./edb/\n rsync -av ./build/extensions/edb/ ./edb/\n rsync -av ./build/lib/edb/ ./edb/\n cp build/postgres/install/stamp build/postgres/\n\n - name: Install edgedb-server\n env:\n BUILD_EXT_MODE: skip\n run: |\n # --no-build-isolation because we have explicitly installed all deps\n # and don't want them to be reinstalled in an \"isolated env\".\n pip install --no-build-isolation --no-deps -e .[test,docs]\n\n # Run the test\n\n - name: Test\n env:\n EDGEDB_DEBUG_DELTA_VALIDATE_REFLECTION: 1\n run: |\n edb test -j2 -v\n\n workflow-notifications:\n if: failure() && github.event_name != 'pull_request'\n name: Notify in Slack on failures\n needs:\n - build\n - test\n runs-on: ubuntu-latest\n permissions:\n actions: 'read'\n steps:\n - name: Slack Workflow Notification\n uses: Gamesight/slack-workflow-status@26a36836c887f260477432e4314ec3490a84f309\n with:\n repo_token: ${{secrets.GITHUB_TOKEN}}\n slack_webhook_url: ${{secrets.ACTIONS_SLACK_WEBHOOK_URL}}\n name: 'Workflow notifications'\n icon_emoji: ':hammer:'\n include_jobs: 'on-failure'\n" }, { "path": ".github/workflows/tests.yml", "content": "name: Tests\n\non:\n push:\n branches:\n - master\n - ci\n - \"release/*\"\n pull_request:\n branches:\n - '**'\n schedule:\n - cron: \"0 */3 * * *\"\n\njobs:\n build:\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - uses: actions/checkout@v4\n with:\n fetch-depth: 50\n submodules: true\n\n - name: Set up Python\n uses: actions/setup-python@v5\n id: setup-python\n with:\n python-version: '3.12.2'\n cache: 'pip'\n cache-dependency-path: |\n pyproject.toml\n\n # The below is technically a lie as we are technically not\n # inside a virtual env, but there is really no reason to bother\n # actually creating and activating one as below works just fine.\n - name: Export $VIRTUAL_ENV\n run: |\n venv=\"$(python -c 'import sys; sys.stdout.write(sys.prefix)')\"\n echo \"VIRTUAL_ENV=${venv}\" >> $GITHUB_ENV\n\n - name: Set up uv cache\n uses: actions/cache@v4\n with:\n path: ~/.cache/uv\n key: uv-cache-${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}\n\n - name: Cached requirements.txt\n uses: actions/cache@v4\n id: requirements-cache\n with:\n path: requirements.txt\n key: edb-requirements-${{ hashFiles('pyproject.toml') }}\n\n - name: Compute requirements.txt\n if: steps.requirements-cache.outputs.cache-hit != 'true'\n run: |\n python -m pip install pip-tools\n pip-compile --no-strip-extras --all-build-deps \\\n --extra test,language-server \\\n --output-file requirements.txt pyproject.toml\n\n - name: Install Python dependencies\n run: |\n python -c \"import sys; print(sys.prefix)\"\n python -m pip install uv~=0.1.0 && uv pip install -U -r requirements.txt\n # 80.9.0 breaks our sphinx, and it keeps sneaking in\n uv pip install setuptools==80.8.0\n\n - name: Compute cache keys and download the running times log\n env:\n GIST_TOKEN: ${{ secrets.CI_BOT_GIST_TOKEN }}\n run: |\n mkdir -p shared-artifacts\n if [ \"$(uname)\" = \"Darwin\" ]; then\n find /usr/lib -type f -name 'lib*' -exec stat -f '%N %z' {} + | sort | shasum -a 256 | cut -d ' ' -f1 > shared-artifacts/lib_cache_key.txt\n else\n find /usr/lib -type f -name 'lib*' -printf '%P %s\\n' | sort | sha256sum | cut -d ' ' -f1 > shared-artifacts/lib_cache_key.txt\n fi\n python setup.py -q ci_helper --type rust >shared-artifacts/rust_cache_key.txt\n python setup.py -q ci_helper --type ext >shared-artifacts/ext_cache_key.txt\n python setup.py -q ci_helper --type parsers >shared-artifacts/parsers_cache_key.txt\n python setup.py -q ci_helper --type postgres >shared-artifacts/postgres_git_rev.txt\n python setup.py -q ci_helper --type libpg_query >shared-artifacts/libpg_query_git_rev.txt\n echo 'f8cd94309eaccbfba5dea7835b88c78377608a37' >shared-artifacts/stolon_git_rev.txt\n python setup.py -q ci_helper --type bootstrap >shared-artifacts/bootstrap_cache_key.txt\n echo POSTGRES_GIT_REV=$(cat shared-artifacts/postgres_git_rev.txt) >> $GITHUB_ENV\n echo LIBPG_QUERY_GIT_REV=$(cat shared-artifacts/libpg_query_git_rev.txt) >> $GITHUB_ENV\n echo STOLON_GIT_REV=$(cat shared-artifacts/stolon_git_rev.txt) >> $GITHUB_ENV\n echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV\n echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV\n\n curl \\\n -H \"Accept: application/vnd.github.v3+json\" \\\n -u edgedb-ci:$GIST_TOKEN \\\n https://api.github.com/gists/8b722a65397f7c4c0df72f5394efa04c \\\n | jq '.files.\"time_stats.csv\".raw_url' \\\n | xargs curl > shared-artifacts/time_stats.csv\n\n - name: Upload shared artifacts\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: shared-artifacts\n path: shared-artifacts\n retention-days: 1\n\n # Restore binary cache\n\n - name: Handle cached Rust extensions\n uses: actions/cache@v4\n id: rust-cache\n with:\n path: build/rust_extensions\n key: edb-rust-v4-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n restore-keys: |\n edb-rust-v4-\n\n - name: Handle cached Cython extensions\n uses: actions/cache@v4\n id: ext-cache\n with:\n path: build/extensions\n key: edb-ext-v6-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Handle cached PostgreSQL build\n uses: actions/cache@v4\n id: postgres-cache\n with:\n path: build/postgres/install\n key: edb-postgres-v3-${{ env.POSTGRES_GIT_REV }}-${{ hashFiles('shared-artifacts/lib_cache_key.txt') }}\n\n - name: Handle cached Stolon build\n uses: actions/cache@v4\n id: stolon-cache\n with:\n path: build/stolon/bin\n key: edb-stolon-v2-${{ env.STOLON_GIT_REV }}\n\n - name: Handle cached libpg_query build\n uses: actions/cache@v4\n id: libpg-query-cache\n with:\n path: edb/pgsql/parser/libpg_query/libpg_query.a\n key: edb-libpg_query-v1-${{ env.LIBPG_QUERY_GIT_REV }}\n\n # Install system dependencies for building\n\n - name: Install system deps\n if: |\n steps.rust-cache.outputs.cache-hit != 'true' ||\n steps.ext-cache.outputs.cache-hit != 'true' ||\n steps.stolon-cache.outputs.cache-hit != 'true' ||\n steps.postgres-cache.outputs.cache-hit != 'true'\n run: |\n sudo apt-get update\n sudo apt-get install -y uuid-dev libreadline-dev bison flex libprotobuf-c-dev\n\n - name: Install Rust toolchain\n if: steps.rust-cache.outputs.cache-hit != 'true'\n uses: dsherret/rust-toolchain-file@v1\n\n # Build Rust extensions\n\n - name: Handle Rust extensions build cache\n uses: actions/cache@v4\n if: steps.rust-cache.outputs.cache-hit != 'true'\n with:\n path: ${{ env.BUILD_TEMP }}/rust/extensions\n key: edb-rust-build-v1-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n restore-keys: |\n edb-rust-build-v1-\n\n - name: Build Rust extensions\n env:\n CARGO_HOME: ${{ env.BUILD_TEMP }}/rust/extensions/cargo_home\n CACHE_HIT: ${{ steps.rust-cache.outputs.cache-hit }}\n run: |\n if [[ \"$CACHE_HIT\" != \"true\" ]]; then\n rm -rf ${BUILD_LIB}\n mkdir -p build/rust_extensions\n rsync -av ./build/rust_extensions/ ${BUILD_LIB}/\n python setup.py -v build_rust\n rsync -av ${BUILD_LIB}/ build/rust_extensions/\n rm -rf ${BUILD_LIB}\n fi\n rsync -av ./build/rust_extensions/edb/ ./edb/\n\n # Build libpg_query\n\n - name: Build libpg_query\n if: |\n steps.libpg-query-cache.outputs.cache-hit != 'true' &&\n steps.ext-cache.outputs.cache-hit != 'true'\n run: |\n python setup.py build_libpg_query\n\n # Build extensions\n\n - name: Handle Cython extensions build cache\n uses: actions/cache@v4\n if: steps.ext-cache.outputs.cache-hit != 'true'\n with:\n path: ${{ env.BUILD_TEMP }}/edb\n key: edb-ext-build-v4-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Build Cython extensions\n env:\n CACHE_HIT: ${{ steps.ext-cache.outputs.cache-hit }}\n BUILD_EXT_MODE: py-only\n run: |\n if [[ \"$CACHE_HIT\" != \"true\" ]]; then\n rm -rf ${BUILD_LIB}\n mkdir -p ./build/extensions\n rsync -av ./build/extensions/ ${BUILD_LIB}/\n BUILD_EXT_MODE=py-only python setup.py -v build_ext\n rsync -av ${BUILD_LIB}/ ./build/extensions/\n rm -rf ${BUILD_LIB}\n fi\n rsync -av ./build/extensions/edb/ ./edb/\n\n # Build parsers\n\n - name: Handle compiled parsers cache\n uses: actions/cache@v4\n id: parsers-cache\n with:\n path: build/lib\n key: edb-parsers-v3-${{ hashFiles('shared-artifacts/parsers_cache_key.txt') }}\n restore-keys: |\n edb-parsers-v3-\n\n - name: Build parsers\n env:\n CACHE_HIT: ${{ steps.parsers-cache.outputs.cache-hit }}\n run: |\n if [[ \"$CACHE_HIT\" != \"true\" ]]; then\n rm -rf ${BUILD_LIB}\n mkdir -p ./build/lib\n rsync -av ./build/lib/ ${BUILD_LIB}/\n python setup.py -v build_parsers\n rsync -av ${BUILD_LIB}/ ./build/lib/\n rm -rf ${BUILD_LIB}\n fi\n rsync -av ./build/lib/edb/ ./edb/\n\n # Build PostgreSQL\n\n - name: Build PostgreSQL\n env:\n CACHE_HIT: ${{ steps.postgres-cache.outputs.cache-hit }}\n run: |\n if [[ \"$CACHE_HIT\" == \"true\" ]]; then\n cp build/postgres/install/stamp build/postgres/\n else\n python setup.py build_postgres\n cp build/postgres/stamp build/postgres/install/\n fi\n\n # Build Stolon\n\n - name: Set up Go\n if: steps.stolon-cache.outputs.cache-hit != 'true'\n uses: actions/setup-go@v2\n with:\n go-version: 1.16\n\n - uses: actions/checkout@v4\n if: steps.stolon-cache.outputs.cache-hit != 'true'\n with:\n repository: edgedb/stolon\n path: build/stolon\n ref: ${{ env.STOLON_GIT_REV }}\n fetch-depth: 0\n submodules: false\n\n - name: Build Stolon\n if: steps.stolon-cache.outputs.cache-hit != 'true'\n run: |\n mkdir -p build/stolon/bin/\n curl -fsSL https://releases.hashicorp.com/consul/1.10.1/consul_1.10.1_linux_amd64.zip | zcat > build/stolon/bin/consul\n chmod +x build/stolon/bin/consul\n cd build/stolon && make\n\n # Install edgedb-server and populate egg-info\n\n - name: Install edgedb-server\n env:\n BUILD_EXT_MODE: skip\n run: |\n # --no-build-isolation because we have explicitly installed all deps\n # and don't want them to be reinstalled in an \"isolated env\".\n pip install --no-build-isolation --no-deps -e .[test,docs]\n\n # Refresh the bootstrap cache\n\n - name: Handle bootstrap cache\n uses: actions/cache@v4\n id: bootstrap-cache\n with:\n path: build/cache\n key: edb-bootstrap-v2-${{ hashFiles('shared-artifacts/bootstrap_cache_key.txt') }}\n restore-keys: |\n edb-bootstrap-v2-\n\n - name: Bootstrap EdgeDB Server\n if: steps.bootstrap-cache.outputs.cache-hit != 'true'\n run: |\n edb server --bootstrap-only\n\n cargo-test:\n needs: build\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - uses: actions/checkout@v4\n with:\n fetch-depth: 50\n submodules: true\n\n - name: Set up Python\n uses: actions/setup-python@v5\n id: setup-python\n with:\n python-version: '3.12.2'\n cache: 'pip'\n cache-dependency-path: |\n pyproject.toml\n\n # The below is technically a lie as we are technically not\n # inside a virtual env, but there is really no reason to bother\n # actually creating and activating one as below works just fine.\n - name: Export $VIRTUAL_ENV\n run: |\n venv=\"$(python -c 'import sys; sys.stdout.write(sys.prefix)')\"\n echo \"VIRTUAL_ENV=${venv}\" >> $GITHUB_ENV\n\n - name: Set up uv cache\n uses: actions/cache@v4\n with:\n path: ~/.cache/uv\n key: uv-cache-${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}\n \n - name: Download requirements.txt\n uses: actions/cache@v4\n with:\n path: requirements.txt\n key: edb-requirements-${{ hashFiles('pyproject.toml') }}\n\n - name: Install Python dependencies\n run: |\n python -m pip install uv~=0.1.0 && uv pip install -U -r requirements.txt\n # 80.9.0 breaks our sphinx, and it keeps sneaking in\n uv pip install setuptools==80.8.0\n\n - name: Download cache key\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: shared-artifacts\n path: shared-artifacts\n\n - name: Generate environment variables\n run: |\n echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV\n\n - name: Handle Rust extensions build cache\n uses: actions/cache@v4\n id: rust-cache\n with:\n path: ${{ env.BUILD_TEMP }}/rust/extensions\n key: edb-rust-build-v1-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n\n - name: Install Rust toolchain\n uses: dsherret/rust-toolchain-file@v1\n\n - name: Cargo test\n env:\n CARGO_TARGET_DIR: ${{ env.BUILD_TEMP }}/rust/extensions\n CARGO_HOME: ${{ env.BUILD_TEMP }}/rust/extensions/cargo_home\n run:\n cargo test --all-features\n\n python-test:\n needs: build\n runs-on: ubuntu-latest\n strategy:\n fail-fast: false\n matrix:\n shard: [\n 1, 2, 3, 4,\n 5, 6, 7, 8,\n 9, 10, 11, 12,\n 13, 14, 15, 16,\n ]\n\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - uses: actions/checkout@v4\n with:\n fetch-depth: 50\n submodules: true\n\n - name: Set up Python\n uses: actions/setup-python@v5\n id: setup-python\n with:\n python-version: '3.12.2'\n cache: 'pip'\n cache-dependency-path: |\n pyproject.toml\n\n # The below is technically a lie as we are technically not\n # inside a virtual env, but there is really no reason to bother\n # actually creating and activating one as below works just fine.\n - name: Export $VIRTUAL_ENV\n run: |\n venv=\"$(python -c 'import sys; sys.stdout.write(sys.prefix)')\"\n echo \"VIRTUAL_ENV=${venv}\" >> $GITHUB_ENV\n\n - name: Set up uv cache\n uses: actions/cache@v4\n with:\n path: ~/.cache/uv\n key: uv-cache-${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}\n \n - name: Download requirements.txt\n uses: actions/cache@v4\n with:\n path: requirements.txt\n key: edb-requirements-${{ hashFiles('pyproject.toml') }}\n\n - name: Install Python dependencies\n run: |\n python -m pip install uv~=0.1.0 && uv pip install -U -r requirements.txt\n # 80.9.0 breaks our sphinx, and it keeps sneaking in\n uv pip install setuptools==80.8.0\n\n # Restore the artifacts and environment variables\n\n - name: Download shared artifacts\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: shared-artifacts\n path: shared-artifacts\n\n - name: Set environment variables\n run: |\n echo POSTGRES_GIT_REV=$(cat shared-artifacts/postgres_git_rev.txt) >> $GITHUB_ENV\n echo STOLON_GIT_REV=$(cat shared-artifacts/stolon_git_rev.txt) >> $GITHUB_ENV\n echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV\n echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV\n\n # Restore build cache\n\n - name: Restore cached Rust extensions\n uses: actions/cache@v4\n id: rust-cache\n with:\n path: build/rust_extensions\n key: edb-rust-v4-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n\n - name: Restore cached Cython extensions\n uses: actions/cache@v4\n id: ext-cache\n with:\n path: build/extensions\n key: edb-ext-v6-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Restore compiled parsers cache\n uses: actions/cache@v4\n id: parsers-cache\n with:\n path: build/lib\n key: edb-parsers-v3-${{ hashFiles('shared-artifacts/parsers_cache_key.txt') }}\n\n - name: Restore cached PostgreSQL build\n uses: actions/cache@v4\n id: postgres-cache\n with:\n path: build/postgres/install\n key: edb-postgres-v3-${{ env.POSTGRES_GIT_REV }}-${{ hashFiles('shared-artifacts/lib_cache_key.txt') }}\n\n - name: Restore cached Stolon build\n uses: actions/cache@v4\n id: stolon-cache\n with:\n path: build/stolon/bin\n key: edb-stolon-v2-${{ env.STOLON_GIT_REV }}\n\n - name: Restore bootstrap cache\n uses: actions/cache@v4\n id: bootstrap-cache\n with:\n path: build/cache\n key: edb-bootstrap-v2-${{ hashFiles('shared-artifacts/bootstrap_cache_key.txt') }}\n\n - name: Stop if we cannot retrieve the cache\n if: |\n steps.rust-cache.outputs.cache-hit != 'true' ||\n steps.ext-cache.outputs.cache-hit != 'true' ||\n steps.parsers-cache.outputs.cache-hit != 'true' ||\n steps.postgres-cache.outputs.cache-hit != 'true' ||\n steps.stolon-cache.outputs.cache-hit != 'true' ||\n steps.bootstrap-cache.outputs.cache-hit != 'true'\n run: |\n echo ::error::Cannot retrieve build cache.\n exit 1\n\n - name: Validate cached binaries\n run: |\n # Validate Stolon\n ./build/stolon/bin/stolon-sentinel --version || exit 1\n ./build/stolon/bin/stolon-keeper --version || exit 1\n ./build/stolon/bin/stolon-proxy --version || exit 1\n\n # Validate PostgreSQL\n ./build/postgres/install/bin/postgres --version || exit 1\n ./build/postgres/install/bin/pg_config --version || exit 1\n\n - name: Restore cache into the source tree\n run: |\n rsync -av ./build/rust_extensions/edb/ ./edb/\n rsync -av ./build/extensions/edb/ ./edb/\n rsync -av ./build/lib/edb/ ./edb/\n cp build/postgres/install/stamp build/postgres/\n\n - name: Install edgedb-server\n env:\n BUILD_EXT_MODE: skip\n run: |\n # --no-build-isolation because we have explicitly installed all deps\n # and don't want them to be reinstalled in an \"isolated env\".\n pip install --no-build-isolation --no-deps -e .[test,docs]\n\n # Run the test\n\n - name: Install Rust toolchain\n uses: dsherret/rust-toolchain-file@v1\n\n - name: Test\n env:\n SHARD: ${{ matrix.shard }}\n EDGEDB_TEST_REPEATS: 1\n run: |\n mkdir -p results/\n cp shared-artifacts/time_stats.csv results/running_times_${SHARD}.csv\n edb test --jobs 2 --verbose --shard ${SHARD}/16 \\\n --running-times-log=results/running_times_${SHARD}.csv \\\n --result-log=results/result_${SHARD}.json\n\n - name: Upload test results\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n if: ${{ always() }}\n with:\n name: python-test-results-${{ matrix.shard }}\n path: results\n retention-days: 1\n\n python-test-list:\n needs: build\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - uses: actions/checkout@v4\n with:\n fetch-depth: 50\n submodules: true\n\n - name: Set up Python\n uses: actions/setup-python@v5\n id: setup-python\n with:\n python-version: '3.12.2'\n cache: 'pip'\n cache-dependency-path: |\n pyproject.toml\n\n # The below is technically a lie as we are technically not\n # inside a virtual env, but there is really no reason to bother\n # actually creating and activating one as below works just fine.\n - name: Export $VIRTUAL_ENV\n run: |\n venv=\"$(python -c 'import sys; sys.stdout.write(sys.prefix)')\"\n echo \"VIRTUAL_ENV=${venv}\" >> $GITHUB_ENV\n\n - name: Set up uv cache\n uses: actions/cache@v4\n with:\n path: ~/.cache/uv\n key: uv-cache-${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}\n \n - name: Download requirements.txt\n uses: actions/cache@v4\n with:\n path: requirements.txt\n key: edb-requirements-${{ hashFiles('pyproject.toml') }}\n\n - name: Install Python dependencies\n run: |\n python -m pip install uv~=0.1.0 && uv pip install -U -r requirements.txt\n # 80.9.0 breaks our sphinx, and it keeps sneaking in\n uv pip install setuptools==80.8.0\n\n # Restore the artifacts and environment variables\n\n - name: Download shared artifacts\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: shared-artifacts\n path: shared-artifacts\n\n - name: Set environment variables\n run: |\n echo POSTGRES_GIT_REV=$(cat shared-artifacts/postgres_git_rev.txt) >> $GITHUB_ENV\n echo STOLON_GIT_REV=$(cat shared-artifacts/stolon_git_rev.txt) >> $GITHUB_ENV\n echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV\n echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV\n\n # Restore build cache\n\n - name: Restore cached Rust extensions\n uses: actions/cache@v4\n id: rust-cache\n with:\n path: build/rust_extensions\n key: edb-rust-v4-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n\n - name: Restore cached Cython extensions\n uses: actions/cache@v4\n id: ext-cache\n with:\n path: build/extensions\n key: edb-ext-v6-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Restore compiled parsers cache\n uses: actions/cache@v4\n id: parsers-cache\n with:\n path: build/lib\n key: edb-parsers-v3-${{ hashFiles('shared-artifacts/parsers_cache_key.txt') }}\n\n - name: Restore cached PostgreSQL build\n uses: actions/cache@v4\n id: postgres-cache\n with:\n path: build/postgres/install\n key: edb-postgres-v3-${{ env.POSTGRES_GIT_REV }}-${{ hashFiles('shared-artifacts/lib_cache_key.txt') }}\n\n - name: Restore cached Stolon build\n uses: actions/cache@v4\n id: stolon-cache\n with:\n path: build/stolon/bin\n key: edb-stolon-v2-${{ env.STOLON_GIT_REV }}\n\n - name: Restore bootstrap cache\n uses: actions/cache@v4\n id: bootstrap-cache\n with:\n path: build/cache\n key: edb-bootstrap-v2-${{ hashFiles('shared-artifacts/bootstrap_cache_key.txt') }}\n\n - name: Stop if we cannot retrieve the cache\n if: |\n steps.rust-cache.outputs.cache-hit != 'true' ||\n steps.ext-cache.outputs.cache-hit != 'true' ||\n steps.parsers-cache.outputs.cache-hit != 'true' ||\n steps.postgres-cache.outputs.cache-hit != 'true' ||\n steps.stolon-cache.outputs.cache-hit != 'true' ||\n steps.bootstrap-cache.outputs.cache-hit != 'true'\n run: |\n echo ::error::Cannot retrieve build cache.\n exit 1\n\n - name: Validate cached binaries\n run: |\n # Validate Stolon\n ./build/stolon/bin/stolon-sentinel --version || exit 1\n ./build/stolon/bin/stolon-keeper --version || exit 1\n ./build/stolon/bin/stolon-proxy --version || exit 1\n\n # Validate PostgreSQL\n ./build/postgres/install/bin/postgres --version || exit 1\n ./build/postgres/install/bin/pg_config --version || exit 1\n\n - name: Restore cache into the source tree\n run: |\n rsync -av ./build/rust_extensions/edb/ ./edb/\n rsync -av ./build/extensions/edb/ ./edb/\n rsync -av ./build/lib/edb/ ./edb/\n cp build/postgres/install/stamp build/postgres/\n\n - name: Install edgedb-server\n env:\n BUILD_EXT_MODE: skip\n run: |\n # --no-build-isolation because we have explicitly installed all deps\n # and don't want them to be reinstalled in an \"isolated env\".\n pip install --no-build-isolation --no-deps -e .[test,docs]\n\n # List tests and upload\n\n - name: Generate complete list of tests for verification\n env:\n SHARD: ${{ matrix.shard }}\n EDGEDB_TEST_REPEATS: 1\n run: |\n edb test --list > shared-artifacts/all_tests.txt\n\n - name: Upload list of tests\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: test-list\n path: shared-artifacts\n retention-days: 1\n\n test-conclusion:\n needs: [cargo-test, python-test, python-test-list]\n runs-on: ubuntu-latest\n if: ${{ always() }}\n steps:\n - name: Set up Python\n uses: actions/setup-python@v5\n with:\n python-version: '3.12.2'\n\n - name: Install Python deps\n run: |\n python -m pip install requests click\n\n - uses: actions/checkout@v4\n with:\n submodules: false\n\n - name: Download python-test results\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n pattern: python-test-results-*\n merge-multiple: true\n path: results\n\n # Render results and exit if they were unsuccessful\n - name: Render results\n run: |\n python edb/tools/test/results.py 'results/result_*.json'\n\n - name: Download shared artifacts\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: shared-artifacts\n path: shared-artifacts\n\n - name: Download test list\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: test-list\n path: shared-artifacts\n\n - name: Merge stats and verify tests completion\n shell: python\n env:\n GIST_TOKEN: ${{ secrets.CI_BOT_GIST_TOKEN }}\n GIT_REF: ${{ github.ref }}\n run: |\n import csv\n import glob\n import io\n import os\n import requests\n\n orig = {}\n new = {}\n all_tests = set()\n with open(\"shared-artifacts/time_stats.csv\") as f:\n for name, t, c in csv.reader(f):\n assert name not in orig, \"duplicate test name in original stats!\"\n orig[name] = (t, int(c))\n\n with open(\"shared-artifacts/all_tests.txt\") as f:\n for line in f:\n assert line not in all_tests, \"duplicate test name in this run!\"\n all_tests.add(line.strip())\n\n for new_file in glob.glob(\"results/running_times_*.csv\"):\n with open(new_file) as f:\n for name, t, c in csv.reader(f):\n if int(c) > orig.get(name, (0, 0))[1]:\n if name.startswith(\"setup::\"):\n new[name] = (t, c)\n else:\n assert name not in new, f\"duplicate test! {name}\"\n new[name] = (t, c)\n all_tests.remove(name)\n\n assert not all_tests, \"Tests not run! \\n\" + \"\\n\".join(all_tests)\n\n if os.environ[\"GIT_REF\"] == \"refs/heads/master\":\n buf = io.StringIO()\n writer = csv.writer(buf)\n orig.update(new)\n for k, v in sorted(orig.items()):\n writer.writerow((k,) + v)\n\n resp = requests.patch(\n \"https://api.github.com/gists/8b722a65397f7c4c0df72f5394efa04c\",\n headers={\"Accept\": \"application/vnd.github.v3+json\"},\n auth=(\"edgedb-ci\", os.environ[\"GIST_TOKEN\"]),\n json={\"files\": {\"time_stats.csv\": {\"content\": buf.getvalue()}}},\n )\n resp.raise_for_status()\n\n\n workflow-notifications:\n if: failure() && github.event_name != 'pull_request'\n name: Notify in Slack on failures\n needs:\n - test-conclusion\n runs-on: ubuntu-latest\n permissions:\n actions: 'read'\n steps:\n - name: Slack Workflow Notification\n uses: Gamesight/slack-workflow-status@26a36836c887f260477432e4314ec3490a84f309\n with:\n repo_token: ${{secrets.GITHUB_TOKEN}}\n slack_webhook_url: ${{secrets.ACTIONS_SLACK_WEBHOOK_URL}}\n name: 'Workflow notifications'\n icon_emoji: ':hammer:'\n include_jobs: 'on-failure'\n" }, { "path": ".github/workflows.src/build.dryrun.tpl.yml", "content": "<% from \"build.inc.yml\" import workflow, workflow_dispatch -%>\nname: Package Build Dry Run\n\non:\n <<- workflow_dispatch() >>\n\njobs:\n <<- workflow(package, targets, [], subdist=\"nightly\") ->>\n" }, { "path": ".github/workflows.src/build.inc.yml", "content": "<% macro workflow_dispatch() %>\n workflow_dispatch:\n inputs:\n gelpkg_ref:\n description: \"gel-pkg git ref used to build the packages\"\n default: \"master\"\n metapkg_ref:\n description: \"metapkg git ref used to build the packages\"\n default: \"master\"\n<%- endmacro %>\n\n<% macro workflow(package, targets, publications, subdist=\"\", publish_all=False) %>\n prep:\n runs-on: ubuntu-latest\n<% if subdist == \"nightly\" %>\n outputs:\n<% for tgt in targets.linux + targets.macos %>\n if_<< tgt.name.replace('-', '_') >>: ${{ steps.scm.outputs.if_<< tgt.name.replace('-', '_') >> }}\n<% endfor %>\n<% endif %>\n steps:\n - uses: actions/checkout@v4\n\n<% if subdist == \"nightly\" %>\n - name: Determine SCM revision\n id: scm\n shell: bash\n run: |\n rev=$(git rev-parse HEAD)\n jq_filter='.packages[] | select(.basename == \"<< package.basename >>\") | select(.architecture == $ARCH) | .version_details.metadata.scm_revision | . as $rev | select(($rev != null) and ($REV | startswith($rev)))'\n<% for tgt in targets.linux + targets.macos %>\n key=\"<< tgt.name >>\"\n val=true\n\n<% if tgt.family == \"debian\" %>\n idx_file=<< tgt.platform_version >>.nightly.json\n url=https://packages.edgedb.com/apt/.jsonindexes/$idx_file\n\n<% elif tgt.family == \"redhat\" %>\n idx_file=el<< tgt.platform_version >>.nightly.json\n url=https://packages.edgedb.com/rpm/.jsonindexes/$idx_file\n\n<% elif tgt.family == \"generic\" %>\n idx_file=<< tgt.platform_version >>-unknown-linux-<< \"{}\".format(tgt.platform_libc) if tgt.platform_libc else \"gnu\" >>.nightly.json\n url=https://packages.edgedb.com/archive/.jsonindexes/$idx_file\n\n<% elif tgt.platform == \"macos\" %>\n idx_file=<< tgt.platform_version >>-apple-darwin.nightly.json\n url=https://packages.edgedb.com/archive/.jsonindexes/$idx_file\n\n<% endif %>\n tmp_file=\"/tmp/$idx_file\"\n\n if [ ! -e \"$tmp_file\" ]; then\n curl --fail -o $tmp_file -s $url || true\n fi\n if [ -e \"$tmp_file\" ]; then\n out=$(< \"$tmp_file\" jq -r --arg REV \"$rev\" --arg ARCH \"<< tgt.arch >>\" \"$jq_filter\")\n if [ -n \"$out\" ]; then\n echo \"Skip rebuilding existing ${key}\"\n val=false\n fi\n fi\n\n echo if_${key//-/_}=\"$val\" >> $GITHUB_OUTPUT\n\n<% endfor %>\n<% endif %>\n\n<%- for tgt in targets.linux %>\n<%- set plat_id = tgt.platform + (\"{}\".format(tgt.platform_libc) if tgt.platform_libc else \"\") + (\"-{}\".format(tgt.platform_version) if tgt.platform_version else \"\") %>\n\n build-<< tgt.name >>:\n runs-on: << tgt.runs_on if tgt.runs_on else \"ubuntu-latest\" >>\n needs: prep\n<% if subdist == \"nightly\" %>\n if: needs.prep.outputs.if_<< tgt.name.replace('-', '_') >> == 'true'\n<% endif %>\n\n steps:\n - name: Build\n uses: docker://ghcr.io/geldata/gelpkg-build-<< plat_id >>:latest\n env:\n PACKAGE: \"<< package.name >>\"\n SRC_REF: \"${{ github.sha }}\"\n PKG_REVISION: \"\"\n <%- if subdist != \"\" %>\n PKG_SUBDIST: \"<< subdist >>\"\n <%- endif %>\n PKG_PLATFORM: \"<< tgt.platform >>\"\n PKG_PLATFORM_VERSION: \"<< tgt.platform_version >>\"\n EXTRA_OPTIMIZATIONS: \"true\"\n <%- if subdist != \"nightly\" %>\n BUILD_IS_RELEASE: \"true\"\n <%- endif %>\n <%- if tgt.family == \"generic\" %>\n BUILD_GENERIC: true\n <%- endif %>\n <%- if tgt.platform_libc %>\n PKG_PLATFORM_LIBC: \"<< tgt.platform_libc >>\"\n <%- endif %>\n METAPKG_GIT_CACHE: disabled\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-<< tgt.name >>\n path: artifacts/<< plat_id >>\n<%- endfor %>\n\n<%- for tgt in targets.macos %>\n<%- set plat_id = tgt.platform + (\"{}\".format(tgt.platform_libc) if tgt.platform_libc else \"\") + (\"-{}\".format(tgt.platform_version) if tgt.platform_version else \"\") %>\n\n build-<< tgt.name >>:\n runs-on: << tgt.runs_on if tgt.runs_on else \"macos-latest\" >>\n needs: prep\n<% if subdist == \"nightly\" %>\n if: needs.prep.outputs.if_<< tgt.name.replace('-', '_') >> == 'true'\n<% endif %>\n\n steps:\n - name: Update Homebrew before installing Rust toolchain\n run: |\n # Homebrew renamed `rustup-init` to `rustup`:\n # https://github.com/Homebrew/homebrew-core/pull/177840\n # But the GitHub Action runner is not updated with this change yet.\n # This caused the later `brew update` in step `Build` to relink Rust\n # toolchain executables, overwriting the custom toolchain installed by\n # `dsherret/rust-toolchain-file`. So let's just run `brew update` early.\n brew update\n\n - uses: actions/checkout@v4\n if: << 'false' if tgt.runs_on and 'self-hosted' in tgt.runs_on else 'true' >>\n with:\n sparse-checkout: |\n rust-toolchain.toml\n sparse-checkout-cone-mode: false\n\n - name: Install Rust toolchain\n uses: dsherret/rust-toolchain-file@v1\n if: << 'false' if tgt.runs_on and 'self-hosted' in tgt.runs_on else 'true' >>\n\n - uses: actions/checkout@v4\n with:\n repository: edgedb/edgedb-pkg\n ref: master\n path: edgedb-pkg\n\n - name: Set up Python\n uses: actions/setup-python@v5\n if: << 'false' if tgt.runs_on and 'self-hosted' in tgt.runs_on else 'true' >>\n with:\n python-version: \"3.12\"\n\n - name: Set up NodeJS\n uses: actions/setup-node@v4\n if: << 'false' if tgt.runs_on and 'self-hosted' in tgt.runs_on else 'true' >>\n with:\n node-version: '20'\n\n - name: Install dependencies\n if: << 'false' if tgt.runs_on and 'self-hosted' in tgt.runs_on else 'true' >>\n run: |\n env HOMEBREW_NO_AUTO_UPDATE=1 brew install libmagic\n\n - name: Install an alias\n # This is probably not strictly needed, but sentencepiece build script reports\n # errors without it.\n if: << 'false' if tgt.runs_on and 'self-hosted' in tgt.runs_on else 'true' >>\n run: |\n printf '#!/bin/sh\\n\\nexec sysctl -n hw.logicalcpu' > /usr/local/bin/nproc\n chmod +x /usr/local/bin/nproc\n\n - name: Build\n env:\n PACKAGE: \"<< package.name >>\"\n SRC_REF: \"${{ github.sha }}\"\n <%- if subdist != \"nightly\" %>\n BUILD_IS_RELEASE: \"true\"\n <%- endif %>\n PKG_REVISION: \"\"\n <%- if subdist != \"\" %>\n PKG_SUBDIST: \"<< subdist >>\"\n <%- endif %>\n PKG_PLATFORM: \"<< tgt.platform >>\"\n PKG_PLATFORM_VERSION: \"<< tgt.platform_version >>\"\n PKG_PLATFORM_ARCH: \"<< tgt.arch if tgt.arch else '' >>\"\n EXTRA_OPTIMIZATIONS: \"true\"\n METAPKG_GIT_CACHE: disabled\n <%- if tgt.family == \"generic\" %>\n BUILD_GENERIC: true\n <%- endif %>\n CMAKE_POLICY_VERSION_MINIMUM: '3.5'\n GEL_PKG_REF: ${{ inputs.gelpkg_ref }}\n METAPKG_REF: ${{ inputs.metapkg_ref }}\n run: |\n edgedb-pkg/integration/macos/build.sh\n\n - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: builds-<< tgt.name >>\n path: artifacts/<< plat_id >>\n<%- endfor %>\n\n<%- if package.name != \"edgedbpkg.edgedb_ls:EdgeDBLanguageServer\" %>\n<%- for tgt in targets.linux %>\n<%- set plat_id = tgt.platform + (\"{}\".format(tgt.platform_libc) if tgt.platform_libc else \"\") + (\"-{}\".format(tgt.platform_version) if tgt.platform_version else \"\") %>\n\n test-<< tgt.name >>:\n needs: [build-<< tgt.name >>]\n runs-on: << tgt.runs_on if tgt.runs_on else \"ubuntu-latest\" >>\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-<< tgt.name >>\n path: artifacts/<< plat_id >>\n\n - name: Test\n uses: docker://ghcr.io/geldata/gelpkg-test-<< plat_id >>:latest\n env:\n <%- if subdist != \"\" %>\n PKG_SUBDIST: \"<< subdist >>\"\n <%- endif %>\n PKG_PLATFORM: \"<< tgt.platform >>\"\n PKG_PLATFORM_VERSION: \"<< tgt.platform_version >>\"\n PKG_PLATFORM_LIBC: \"<< tgt.platform_libc >>\"\n PKG_TEST_SELECT: \"<< package.test.select >>\"\n PKG_TEST_EXCLUDE: \"<< package.test.exclude >>\"\n PKG_TEST_FILES: \"<< package.test.files >> << tgt.test.files >>\"\n # edb test with -j higher than 1 seems to result in workflow\n # jobs getting killed arbitrarily by Github.\n PKG_TEST_JOBS: << 0 if tgt.runs_on and 'self-hosted' in tgt.runs_on else 1 >>\n<%- endfor %>\n\n<%- for tgt in targets.macos %>\n<%- set plat_id = tgt.platform + (\"{}\".format(tgt.platform_libc) if tgt.platform_libc else \"\") + (\"-{}\".format(tgt.platform_version) if tgt.platform_version else \"\") %>\n\n test-<< tgt.name >>:\n needs: [build-<< tgt.name >>]\n runs-on: << tgt.runs_on if tgt.runs_on else \"macos-latest\" >>\n\n steps:\n - uses: actions/checkout@v4\n with:\n repository: edgedb/edgedb-pkg\n ref: master\n path: edgedb-pkg\n\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-<< tgt.name >>\n path: artifacts/<< plat_id >>\n\n - name: Test\n env:\n <%- if subdist != \"\" %>\n PKG_SUBDIST: \"<< subdist >>\"\n <%- endif %>\n PKG_PLATFORM: \"<< tgt.platform >>\"\n PKG_PLATFORM_VERSION: \"<< tgt.platform_version >>\"\n PKG_TEST_SELECT: \"<< package.test.select >>\"\n PKG_TEST_EXCLUDE: \"<< package.test.exclude >>\"\n PKG_TEST_FILES: \"<< package.test.files >> << tgt.test.files >>\"\n run: |\n <%- if tgt.platform_version == \"x86_64\" %>\n # Bump shmmax and shmall to avoid test failures.\n sudo sysctl -w kern.sysv.shmmax=12582912\n sudo sysctl -w kern.sysv.shmall=12582912\n <%- endif %>\n edgedb-pkg/integration/macos/test.sh\n<%- endfor %>\n<%- endif %>\n\n<%- if publish_all %>\n collect:\n needs:\n <%- for tgt in targets.linux + targets.macos %>\n - test-<< tgt.name >>\n <%- endfor %>\n runs-on: ubuntu-latest\n steps:\n - run: echo 'All build+tests passed, ready to publish now!'\n<%- endif %>\n\n<%- for tgt in targets.linux %>\n<%- set plat_id = tgt.platform + (\"{}\".format(tgt.platform_libc) if tgt.platform_libc else \"\") + (\"-{}\".format(tgt.platform_version) if tgt.platform_version else \"\") %>\n<%- for publish in publications %>\n\n publish<< publish.suffix>>-<< tgt.name >>:\n needs: [<% if publish_all %>collect<% elif package.name != \"edgedbpkg.edgedb_ls:EdgeDBLanguageServer\" %>test-<< tgt.name >><% else %>build-<< tgt.name >><% endif %>]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-<< tgt.name >>\n path: artifacts/<< plat_id >>\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n <%- if subdist != \"\" %>\n PKG_SUBDIST: \"<< subdist >>\"\n <%- endif %>\n <%- if publish.server != \"\" %>\n PACKAGE_SERVER: << publish.server >>\n <%- endif %>\n PKG_PLATFORM: \"<< tgt.platform >>\"\n PKG_PLATFORM_VERSION: \"<< tgt.platform_version >>\"\n PKG_PLATFORM_LIBC: \"<< tgt.platform_libc >>\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n\n check-published<>-<< tgt.name >>:\n needs: [publish<< publish.suffix >>-<< tgt.name >>]\n runs-on: << tgt.runs_on if tgt.runs_on else \"ubuntu-latest\" >>\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-<< tgt.name >>\n path: artifacts/<< plat_id >>\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: << plat_id >>\n\n - name: Test Published\n uses: docker://ghcr.io/geldata/gelpkg-testpublished-<< plat_id >>:latest\n env:\n PKG_NAME: \"${{ steps.describe.outputs.name }}\"\n <%- if subdist != \"\" %>\n PKG_SUBDIST: \"<< subdist >>\"\n <%- endif %>\n <%- if publish.server != \"\" %>\n PACKAGE_SERVER: << publish.server >>\n <%- endif %>\n PKG_PLATFORM: \"<< tgt.platform >>\"\n PKG_PLATFORM_VERSION: \"<< tgt.platform_version >>\"\n PKG_INSTALL_REF: \"${{ steps.describe.outputs.install-ref }}\"\n PKG_VERSION_SLOT: \"${{ steps.describe.outputs.version-slot }}\"\n\n outputs:\n version-slot: ${{ steps.describe.outputs.version-slot }}\n version-core: ${{ steps.describe.outputs.version-core }}\n catalog-version: ${{ steps.describe.outputs.catalog-version }}\n<%- endfor %>\n<%- endfor %>\n\n<%- if publications %>\n<%- for tgt in targets.macos %>\n<%- set plat_id = tgt.platform + (\"{}\".format(tgt.platform_libc) if tgt.platform_libc else \"\") + (\"-{}\".format(tgt.platform_version) if tgt.platform_version else \"\") %>\n\n publish-<< tgt.name >>:\n needs: [<% if publish_all %>collect<% elif package.name != \"edgedbpkg.edgedb_ls:EdgeDBLanguageServer\" %>test-<< tgt.name >><% else %>build-<< tgt.name >><% endif %>]\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: builds-<< tgt.name >>\n path: artifacts/<< plat_id >>\n\n - uses: actions/checkout@v4\n with:\n repository: edgedb/edgedb-pkg\n ref: master\n path: edgedb-pkg\n\n - name: Describe\n id: describe\n uses: edgedb/edgedb-pkg/integration/actions/describe-artifact@master\n with:\n target: << plat_id >>\n\n - name: Publish\n uses: docker://ghcr.io/geldata/gelpkg-upload-linux-x86_64:latest\n env:\n <%- if subdist != \"\" %>\n PKG_SUBDIST: \"<< subdist >>\"\n <%- endif %>\n PKG_PLATFORM: \"<< tgt.platform >>\"\n PKG_PLATFORM_VERSION: \"<< tgt.platform_version >>\"\n PACKAGE_UPLOAD_SSH_KEY: \"${{ secrets.PACKAGE_UPLOAD_SSH_KEY }}\"\n<%- endfor %>\n<%- endif %>\n\n<%- set docker_tgts = targets.linux | selectattr(\"docker_arch\") | list %>\n<%- if docker_tgts and publications %>\n<%- set pub_outputs = \"needs.check-published-\" + (docker_tgts|first)[\"name\"] + \".outputs\" %>\n\n publish-docker:\n needs:\n <%- for tgt in docker_tgts %>\n - check-published-<< tgt.name >>\n <%- endfor %>\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/checkout@v4\n with:\n repository: geldata/gel-docker\n ref: master\n path: dockerfile\n\n - name: Login to Docker Hub\n uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0\n with:\n username: ${{ secrets.DOCKER_USERNAME }}\n password: ${{ secrets.DOCKER_PASSWORD }}\n\n - name: Login to GitHub Container Registry\n uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0\n with:\n registry: ghcr.io\n username: \"edgedb-ci\"\n password: ${{ secrets.GITHUB_CI_BOT_TOKEN }}\n\n - env:\n VERSION_SLOT: \"${{ << pub_outputs >>.version-slot }}\"\n VERSION_CORE: \"${{ << pub_outputs >>.version-core }}\"\n CATALOG_VERSION: \"${{ << pub_outputs >>.catalog-version }}\"\n PKG_SUBDIST: \"<< subdist >>\"\n id: tags\n run: |\n set -e\n\n url='https://registry.hub.docker.com/v2/repositories/geldata/gel/tags?page_size=100'\n repo_tags=$(\n while [ -n \"$url\" ]; do\n resp=$(curl -L -s \"$url\")\n url=$(echo \"$resp\" | jq -r \".next\")\n if [ \"$url\" = \"null\" ] || [ -z \"$url\" ]; then\n break\n fi\n echo \"$resp\" | jq -r '.\"results\"[][\"name\"]'\n done | grep \"^[[:digit:]]\\+.*\" | grep -v \"alpha\\|beta\\|rc\" || :\n )\n\n tags=()\n\n if [ \"$PKG_SUBDIST\" = \"nightly\" ]; then\n tags+=(\n \"nightly\"\n \"nightly_${VERSION_SLOT}_cv${CATALOG_VERSION}\"\n )\n else\n tags+=( \"$VERSION_CORE\" )\n\n top=$(printf \"%s\\n%s\\n\" \"$VERSION_CORE\" \"$repo_tags\" \\\n | grep \"^${VERSION_SLOT}[\\.-]\" \\\n | sort --version-sort --reverse | head -n 1)\n if [ \"$top\" == \"$VERSION_CORE\" ]; then\n tags+=( \"$VERSION_SLOT\" )\n fi\n\n if [ -z \"$PKG_SUBDIST\" ]; then\n top=$(printf \"%s\\n%s\\n\" \"$VERSION_CORE\" \"$repo_tags\" \\\n | sort --version-sort --reverse | head -n 1)\n if [ \"$top\" == \"$VERSION_CORE\" ]; then\n tags+=( \"latest\" )\n fi\n fi\n fi\n\n fq_tags=()\n images=(\"geldata/gel\" \"ghcr.io/geldata/gel\")\n\n for image in \"${images[@]}\"; do\n fq_tags+=(\"${tags[@]/#/${image}:}\")\n done\n\n IFS=,\n echo \"tags=${fq_tags[*]}\" >> $GITHUB_OUTPUT\n\n - name: Set up QEMU\n uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0\n\n - name: Set up Docker Buildx\n uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0\n\n - name: Build and Publish Docker Image\n uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6.10.0\n with:\n push: true\n provenance: mode=max\n tags: \"${{ steps.tags.outputs.tags }}\"\n context: dockerfile\n build-args: |\n version=${{ << pub_outputs >>.version-slot }}\n exact_version=${{ << pub_outputs >>.version-core }}\n <%- if subdist != \"\" %>\n subdist=<< subdist >>\n <%- endif %>\n platforms: << docker_tgts|map(attribute=\"docker_arch\")|join(\",\") >>\n<%- endif %>\n\n workflow-notifications:\n if: failure() && github.event_name != 'pull_request'\n name: Notify in Slack on failures\n\n needs:\n - prep\n <%- if publish_all %>\n - collect\n <%- else %>\n <%- endif %>\n <%- for tgt in targets.linux %>\n - build-<< tgt.name >>\n <%- if package.name != \"edgedbpkg.edgedb_ls:EdgeDBLanguageServer\" %>\n - test-<< tgt.name >>\n <%- endif %>\n <%- for publish in publications %>\n - publish<< publish.suffix>>-<< tgt.name >>\n - check-published<< publish.suffix>>-<< tgt.name >>\n <%- endfor %>\n <%- endfor %>\n <%- for tgt in targets.macos %>\n - build-<< tgt.name >>\n <%- if package.name != \"edgedbpkg.edgedb_ls:EdgeDBLanguageServer\" %>\n - test-<< tgt.name >>\n <%- endif %>\n <%- for publish in publications %>\n - publish<< publish.suffix>>-<< tgt.name >>\n <%- endfor %>\n <%- endfor %>\n <%- if docker_tgts and publications %>\n - publish-docker\n <%- endif %>\n runs-on: ubuntu-latest\n permissions:\n actions: 'read'\n steps:\n - name: Slack Workflow Notification\n uses: Gamesight/slack-workflow-status@26a36836c887f260477432e4314ec3490a84f309\n with:\n repo_token: ${{secrets.GITHUB_TOKEN}}\n slack_webhook_url: ${{secrets.ACTIONS_SLACK_WEBHOOK_URL}}\n name: 'Workflow notifications'\n icon_emoji: ':hammer:'\n include_jobs: 'on-failure'\n\n<%- endmacro %>\n" }, { "path": ".github/workflows.src/build.ls-nightly.tpl.yml", "content": "<% from \"build.inc.yml\" import workflow, workflow_dispatch -%>\nname: 'ls: Build and Publish Nightly Packages'\n\non:\n schedule:\n - cron: \"0 1 * * *\"\n <<- workflow_dispatch() >>\n push:\n branches:\n - nightly\n\njobs:\n <<- workflow(package, targets, publications, subdist=\"nightly\") ->>\n" }, { "path": ".github/workflows.src/build.ls.targets.yml", "content": "publications:\n - name: prod\n suffix: \"\"\n server: sftp://uploader@package-upload.edgedb.net:22/\n\npackage:\n name: edgedbpkg.edgedb_ls:EdgeDBLanguageServer\n basename: gel-ls\n tests:\n files: \"test_language_server.py\"\n\ntargets:\n linux:\n - name: linux-x86_64\n arch: x86_64\n platform: linux\n platform_version: x86_64\n family: generic\n runs_on: [self-hosted, linux, x64]\n - name: linux-aarch64\n arch: aarch64\n platform: linux\n platform_version: aarch64\n family: generic\n runs_on: [self-hosted, linux, arm64]\n - name: linuxmusl-x86_64\n arch: x86_64\n platform: linux\n platform_version: x86_64\n platform_libc: musl\n family: generic\n runs_on: [self-hosted, linux, x64]\n - name: linuxmusl-aarch64\n arch: aarch64\n platform: linux\n platform_version: aarch64\n platform_libc: musl\n family: generic\n runs_on: [self-hosted, linux, arm64]\n\n macos:\n - name: macos-x86_64\n arch: x86_64\n platform: macos\n platform_version: x86_64\n family: generic\n runs_on: [macos-13]\n - name: macos-aarch64\n arch: aarch64\n platform: macos\n platform_version: aarch64\n family: generic\n runs_on: [macos-14]\n" }, { "path": ".github/workflows.src/build.nightly.tpl.yml", "content": "<% from \"build.inc.yml\" import workflow, workflow_dispatch -%>\nname: Build Test and Publish Nightly Packages\n\non:\n schedule:\n - cron: \"0 1 * * *\"\n <<- workflow_dispatch() >>\n push:\n branches:\n - nightly\n\njobs:\n <<- workflow(package, targets, publications, subdist=\"nightly\") ->>\n" }, { "path": ".github/workflows.src/build.release.tpl.yml", "content": "<% from \"build.inc.yml\" import workflow, workflow_dispatch -%>\nname: Build Test and Publish a Release\n\non:\n <<- workflow_dispatch() >>\n\njobs:\n <<- workflow(package, targets, publications, subdist=\"\", publish_all=True) ->>\n" }, { "path": ".github/workflows.src/build.targets.yml", "content": "publications:\n - name: prod\n suffix: \"\"\n server: sftp://uploader@package-upload.edgedb.net:22/\n\npackage:\n name: \"edgedbpkg.edgedb:Gel\"\n basename: gel-server\n\ntargets:\n linux:\n - name: debian-buster-x86_64\n arch: x86_64\n platform: debian\n platform_version: buster\n family: debian\n runs_on: [package-builder, self-hosted, linux, x64]\n - name: debian-buster-aarch64\n arch: aarch64\n platform: debian\n platform_version: buster\n family: debian\n runs_on: [package-builder, self-hosted, linux, arm64]\n - name: debian-bullseye-x86_64\n arch: x86_64\n platform: debian\n platform_version: bullseye\n family: debian\n runs_on: [package-builder, self-hosted, linux, x64]\n - name: debian-bullseye-aarch64\n arch: aarch64\n platform: debian\n platform_version: bullseye\n family: debian\n runs_on: [package-builder, self-hosted, linux, arm64]\n - name: debian-bookworm-x86_64\n arch: x86_64\n platform: debian\n platform_version: bookworm\n family: debian\n runs_on: [package-builder, self-hosted, linux, x64]\n docker_arch: linux/amd64\n - name: debian-bookworm-aarch64\n arch: aarch64\n platform: debian\n platform_version: bookworm\n family: debian\n runs_on: [package-builder, self-hosted, linux, arm64]\n docker_arch: linux/arm64\n - name: ubuntu-focal-x86_64\n arch: x86_64\n platform: ubuntu\n platform_version: focal\n family: debian\n runs_on: [package-builder, self-hosted, linux, x64]\n - name: ubuntu-focal-aarch64\n arch: aarch64\n platform: ubuntu\n platform_version: focal\n family: debian\n runs_on: [package-builder, self-hosted, linux, arm64]\n - name: ubuntu-jammy-x86_64\n arch: x86_64\n platform: ubuntu\n platform_version: jammy\n family: debian\n runs_on: [package-builder, self-hosted, linux, x64]\n - name: ubuntu-jammy-aarch64\n arch: aarch64\n platform: ubuntu\n platform_version: jammy\n family: debian\n runs_on: [package-builder, self-hosted, linux, arm64]\n - name: ubuntu-noble-x86_64\n arch: x86_64\n platform: ubuntu\n platform_version: noble\n family: debian\n runs_on: [package-builder, self-hosted, linux, x64]\n - name: ubuntu-noble-aarch64\n arch: aarch64\n platform: ubuntu\n platform_version: noble\n family: debian\n runs_on: [package-builder, self-hosted, linux, arm64]\n - name: centos-8-x86_64\n arch: x86_64\n platform: centos\n platform_version: 8\n family: redhat\n runs_on: [package-builder, self-hosted, linux, x64]\n - name: centos-8-aarch64\n arch: aarch64\n platform: centos\n platform_version: 8\n family: redhat\n runs_on: [package-builder, self-hosted, linux, arm64]\n - name: rockylinux-9-x86_64\n arch: x86_64\n platform: rockylinux\n platform_version: 9\n family: redhat\n runs_on: [package-builder, self-hosted, linux, x64]\n - name: rockylinux-9-aarch64\n arch: aarch64\n platform: rockylinux\n platform_version: 9\n family: redhat\n runs_on: [package-builder, self-hosted, linux, arm64]\n - name: linux-x86_64\n arch: x86_64\n platform: linux\n platform_version: x86_64\n family: generic\n runs_on: [package-builder, self-hosted, linux, x64]\n - name: linux-aarch64\n arch: aarch64\n platform: linux\n platform_version: aarch64\n family: generic\n runs_on: [package-builder, self-hosted, linux, arm64]\n - name: linuxmusl-x86_64\n arch: x86_64\n platform: linux\n platform_version: x86_64\n platform_libc: musl\n family: generic\n runs_on: [package-builder, self-hosted, linux, x64]\n - name: linuxmusl-aarch64\n arch: aarch64\n platform: linux\n platform_version: aarch64\n platform_libc: musl\n family: generic\n runs_on: [package-builder, self-hosted, linux, arm64]\n\n macos:\n - name: macos-x86_64\n arch: x86_64\n platform: macos\n platform_version: x86_64\n family: generic\n runs_on: [macos-13]\n # Run fewer tests on x86_64, since the test runner is very slow.\n test:\n files: >\n test_dump*.py test_backend_*.py test_database.py\n test_server_*.py test_edgeql_ddl.py test_session.py\n - name: macos-aarch64\n arch: aarch64\n platform: macos\n platform_version: aarch64\n family: generic\n runs_on: [macos-14]\n" }, { "path": ".github/workflows.src/build.testing.tpl.yml", "content": "<% from \"build.inc.yml\" import workflow, workflow_dispatch -%>\nname: Build Test and Publish a Testing Release\n\non:\n <<- workflow_dispatch() >>\n\njobs:\n <<- workflow(package, targets, publications, subdist=\"testing\", publish_all=True) ->>\n" }, { "path": ".github/workflows.src/render.py", "content": "#!/usr/bin/env python3\n\nimport argparse\nimport pathlib\nimport sys\n\nimport jinja2\nimport yaml\n\n\nenv = jinja2.Environment(\n variable_start_string='<<',\n variable_end_string='>>',\n block_start_string='<%',\n block_end_string='%>',\n loader=jinja2.FileSystemLoader(pathlib.Path(__file__).parent),\n)\n\n\ndef die(msg):\n print(msg, file=sys.stderr)\n sys.exit(1)\n\n\ndef _expand_test_spec(target):\n if \"test\" not in target:\n target[\"test\"] = {\n \"include\": \"\",\n \"exclude\": \"\",\n \"files\": \"\"\n }\n\n for key in {\"include\", \"exclude\", \"files\"}:\n if key not in target[\"test\"]:\n target[\"test\"][key] = \"\"\n\n\ndef _render(tpl_path, data):\n with open(tpl_path) as f:\n tpl = env.from_string(f.read())\n\n return tpl.render(**data)\n\n\ndef main():\n parser = argparse.ArgumentParser()\n parser.add_argument('--workflow', choices=[\"build\", \"test\"], required=True)\n parser.add_argument('template')\n parser.add_argument('datafile')\n\n args = parser.parse_args()\n\n tplfile = f'{args.template}.tpl.yml'\n path = pathlib.Path(__file__).parent / tplfile\n\n if not path.exists():\n die(f'template does not exist: {tplfile}')\n\n datapath = pathlib.Path(__file__).parent / args.datafile\n\n if datapath.exists():\n with open(datapath) as f:\n data = yaml.load(f, Loader=yaml.SafeLoader)\n else:\n data = {}\n\n if args.workflow == \"build\":\n package = data.get(\"package\")\n if not package or not isinstance(package, dict):\n die(f\"invalid package: specification in {datapath}\")\n if not package.get(\"name\"):\n die(f\"missing package.name in {datapath}\")\n\n _expand_test_spec(package)\n\n targets = data.get(\"targets\")\n if not targets or not isinstance(targets, dict):\n die(f\"invalid targets: specification in {datapath}\")\n\n for target_list in targets.values():\n for target in target_list:\n _expand_test_spec(target)\n\n output = _render(path, data)\n\n target = (\n pathlib.Path(__file__).parent.parent\n / 'workflows'\n / f'{args.template}.yml'\n )\n with open(target, 'w') as f:\n print(output, file=f)\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": ".github/workflows.src/tests.ha.targets.yml", "content": "data:\n" }, { "path": ".github/workflows.src/tests.ha.tpl.yml", "content": "<% from \"tests.inc.yml\" import build, calc_cache_key, restore_cache -%>\nname: High Availability Tests\n\non:\n workflow_dispatch:\n inputs: {}\n workflow_run:\n workflows: [\"Tests\"]\n types:\n - completed\n\njobs:\n build:\n runs-on: ubuntu-latest\n if: github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch'\n\n steps:\n <%- call build() -%>\n # Our HA tests currently only work on Postgres 14 (see #6332),\n # so check it out before we compute our build cache keys.\n - name: Switch back to Postgres 14\n shell: bash\n run: |\n set -e\n cd postgres\n # Fetch postgres 14, since the clone was shallow\n git fetch origin REL_14_8 --depth=1\n # For whatever reason the tag doesn't get fetched, so find it\n # at FETCH_HEAD\n git checkout FETCH_HEAD\n\n - name: Compute cache keys\n env:\n GIST_TOKEN: ${{ secrets.CI_BOT_GIST_TOKEN }}\n run: |\n << calc_cache_key()|indent >>\n <%- endcall %>\n\n ha-test:\n needs: build\n runs-on: ubuntu-latest\n\n steps:\n <<- restore_cache() >>\n\n # Run the test\n\n - name: Test\n env:\n SHARD: ${{ matrix.shard }}\n EDGEDB_TEST_HA: 1\n EDGEDB_TEST_CONSUL_PATH: build/stolon/bin/consul\n EDGEDB_TEST_STOLON_CTL: build/stolon/bin/stolonctl\n EDGEDB_TEST_STOLON_SENTINEL: build/stolon/bin/stolon-sentinel\n EDGEDB_TEST_STOLON_KEEPER: build/stolon/bin/stolon-keeper\n run: |\n edb test -j1 -v -k test_ha_\n\n\n workflow-notifications:\n if: failure() && github.event_name != 'pull_request'\n name: Notify in Slack on failures\n needs:\n - build\n - ha-test\n runs-on: ubuntu-latest\n permissions:\n actions: 'read'\n steps:\n - name: Slack Workflow Notification\n uses: Gamesight/slack-workflow-status@26a36836c887f260477432e4314ec3490a84f309\n with:\n repo_token: ${{secrets.GITHUB_TOKEN}}\n slack_webhook_url: ${{secrets.ACTIONS_SLACK_WEBHOOK_URL}}\n name: 'Workflow notifications'\n icon_emoji: ':hammer:'\n include_jobs: 'on-failure'\n" }, { "path": ".github/workflows.src/tests.inc.yml", "content": "<% macro init(ref='') -%>\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n<%- if ref != \"\" %>\n ref: << ref >>\n<%- endif %>\n\n - uses: actions/checkout@v4\n with:\n fetch-depth: 50\n submodules: true\n<%- if ref != \"\" %>\n ref: << ref >>\n<%- endif %>\n\n - name: Set up Python\n uses: actions/setup-python@v5\n id: setup-python\n with:\n python-version: '3.12.2'\n cache: 'pip'\n cache-dependency-path: |\n pyproject.toml\n\n # The below is technically a lie as we are technically not\n # inside a virtual env, but there is really no reason to bother\n # actually creating and activating one as below works just fine.\n - name: Export $VIRTUAL_ENV\n run: |\n venv=\"$(python -c 'import sys; sys.stdout.write(sys.prefix)')\"\n echo \"VIRTUAL_ENV=${venv}\" >> $GITHUB_ENV\n\n - name: Set up uv cache\n uses: actions/cache@v4\n with:\n path: ~/.cache/uv\n key: uv-cache-${{ runner.os }}-py-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}\n\n<%- endmacro %>\n\n<% macro build(ref=\"\") %>\n << init(ref) >>\n\n - name: Cached requirements.txt\n uses: actions/cache@v4\n id: requirements-cache\n with:\n path: requirements.txt\n key: edb-requirements-${{ hashFiles('pyproject.toml') }}\n\n - name: Compute requirements.txt\n if: steps.requirements-cache.outputs.cache-hit != 'true'\n run: |\n python -m pip install pip-tools\n pip-compile --no-strip-extras --all-build-deps \\\n --extra test,language-server \\\n --output-file requirements.txt pyproject.toml\n\n - name: Install Python dependencies\n run: |\n python -c \"import sys; print(sys.prefix)\"\n python -m pip install uv~=0.1.0 && uv pip install -U -r requirements.txt\n # 80.9.0 breaks our sphinx, and it keeps sneaking in\n uv pip install setuptools==80.8.0\n\n << caller() >>\n\n - name: Upload shared artifacts\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: shared-artifacts\n path: shared-artifacts\n retention-days: 1\n\n # Restore binary cache\n\n - name: Handle cached Rust extensions\n uses: actions/cache@v4\n id: rust-cache\n with:\n path: build/rust_extensions\n key: edb-rust-v4-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n restore-keys: |\n edb-rust-v4-\n\n - name: Handle cached Cython extensions\n uses: actions/cache@v4\n id: ext-cache\n with:\n path: build/extensions\n key: edb-ext-v6-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Handle cached PostgreSQL build\n uses: actions/cache@v4\n id: postgres-cache\n with:\n path: build/postgres/install\n key: edb-postgres-v3-${{ env.POSTGRES_GIT_REV }}-${{ hashFiles('shared-artifacts/lib_cache_key.txt') }}\n\n - name: Handle cached Stolon build\n uses: actions/cache@v4\n id: stolon-cache\n with:\n path: build/stolon/bin\n key: edb-stolon-v2-${{ env.STOLON_GIT_REV }}\n\n - name: Handle cached libpg_query build\n uses: actions/cache@v4\n id: libpg-query-cache\n with:\n path: edb/pgsql/parser/libpg_query/libpg_query.a\n key: edb-libpg_query-v1-${{ env.LIBPG_QUERY_GIT_REV }}\n\n # Install system dependencies for building\n\n - name: Install system deps\n if: |\n steps.rust-cache.outputs.cache-hit != 'true' ||\n steps.ext-cache.outputs.cache-hit != 'true' ||\n steps.stolon-cache.outputs.cache-hit != 'true' ||\n steps.postgres-cache.outputs.cache-hit != 'true'\n run: |\n sudo apt-get update\n sudo apt-get install -y uuid-dev libreadline-dev bison flex libprotobuf-c-dev\n\n - name: Install Rust toolchain\n if: steps.rust-cache.outputs.cache-hit != 'true'\n uses: dsherret/rust-toolchain-file@v1\n\n # Build Rust extensions\n\n - name: Handle Rust extensions build cache\n uses: actions/cache@v4\n if: steps.rust-cache.outputs.cache-hit != 'true'\n with:\n path: ${{ env.BUILD_TEMP }}/rust/extensions\n key: edb-rust-build-v1-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n restore-keys: |\n edb-rust-build-v1-\n\n - name: Build Rust extensions\n env:\n CARGO_HOME: ${{ env.BUILD_TEMP }}/rust/extensions/cargo_home\n CACHE_HIT: ${{ steps.rust-cache.outputs.cache-hit }}\n run: |\n if [[ \"$CACHE_HIT\" != \"true\" ]]; then\n rm -rf ${BUILD_LIB}\n mkdir -p build/rust_extensions\n rsync -av ./build/rust_extensions/ ${BUILD_LIB}/\n python setup.py -v build_rust\n rsync -av ${BUILD_LIB}/ build/rust_extensions/\n rm -rf ${BUILD_LIB}\n fi\n rsync -av ./build/rust_extensions/edb/ ./edb/\n\n # Build libpg_query\n\n - name: Build libpg_query\n if: |\n steps.libpg-query-cache.outputs.cache-hit != 'true' &&\n steps.ext-cache.outputs.cache-hit != 'true'\n run: |\n python setup.py build_libpg_query\n\n # Build extensions\n\n - name: Handle Cython extensions build cache\n uses: actions/cache@v4\n if: steps.ext-cache.outputs.cache-hit != 'true'\n with:\n path: ${{ env.BUILD_TEMP }}/edb\n key: edb-ext-build-v4-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Build Cython extensions\n env:\n CACHE_HIT: ${{ steps.ext-cache.outputs.cache-hit }}\n BUILD_EXT_MODE: py-only\n run: |\n if [[ \"$CACHE_HIT\" != \"true\" ]]; then\n rm -rf ${BUILD_LIB}\n mkdir -p ./build/extensions\n rsync -av ./build/extensions/ ${BUILD_LIB}/\n BUILD_EXT_MODE=py-only python setup.py -v build_ext\n rsync -av ${BUILD_LIB}/ ./build/extensions/\n rm -rf ${BUILD_LIB}\n fi\n rsync -av ./build/extensions/edb/ ./edb/\n\n # Build parsers\n\n - name: Handle compiled parsers cache\n uses: actions/cache@v4\n id: parsers-cache\n with:\n path: build/lib\n key: edb-parsers-v3-${{ hashFiles('shared-artifacts/parsers_cache_key.txt') }}\n restore-keys: |\n edb-parsers-v3-\n\n - name: Build parsers\n env:\n CACHE_HIT: ${{ steps.parsers-cache.outputs.cache-hit }}\n run: |\n if [[ \"$CACHE_HIT\" != \"true\" ]]; then\n rm -rf ${BUILD_LIB}\n mkdir -p ./build/lib\n rsync -av ./build/lib/ ${BUILD_LIB}/\n python setup.py -v build_parsers\n rsync -av ${BUILD_LIB}/ ./build/lib/\n rm -rf ${BUILD_LIB}\n fi\n rsync -av ./build/lib/edb/ ./edb/\n\n # Build PostgreSQL\n\n - name: Build PostgreSQL\n env:\n CACHE_HIT: ${{ steps.postgres-cache.outputs.cache-hit }}\n run: |\n if [[ \"$CACHE_HIT\" == \"true\" ]]; then\n cp build/postgres/install/stamp build/postgres/\n else\n python setup.py build_postgres\n cp build/postgres/stamp build/postgres/install/\n fi\n\n # Build Stolon\n\n - name: Set up Go\n if: steps.stolon-cache.outputs.cache-hit != 'true'\n uses: actions/setup-go@v2\n with:\n go-version: 1.16\n\n - uses: actions/checkout@v4\n if: steps.stolon-cache.outputs.cache-hit != 'true'\n with:\n repository: edgedb/stolon\n path: build/stolon\n ref: ${{ env.STOLON_GIT_REV }}\n fetch-depth: 0\n submodules: false\n\n - name: Build Stolon\n if: steps.stolon-cache.outputs.cache-hit != 'true'\n run: |\n mkdir -p build/stolon/bin/\n curl -fsSL https://releases.hashicorp.com/consul/1.10.1/consul_1.10.1_linux_amd64.zip | zcat > build/stolon/bin/consul\n chmod +x build/stolon/bin/consul\n cd build/stolon && make\n\n # Install edgedb-server and populate egg-info\n\n - name: Install edgedb-server\n env:\n BUILD_EXT_MODE: skip\n run: |\n # --no-build-isolation because we have explicitly installed all deps\n # and don't want them to be reinstalled in an \"isolated env\".\n pip install --no-build-isolation --no-deps -e .[test,docs]\n\n # Refresh the bootstrap cache\n\n - name: Handle bootstrap cache\n uses: actions/cache@v4\n id: bootstrap-cache\n with:\n path: build/cache\n key: edb-bootstrap-v2-${{ hashFiles('shared-artifacts/bootstrap_cache_key.txt') }}\n restore-keys: |\n edb-bootstrap-v2-\n\n - name: Bootstrap EdgeDB Server\n if: steps.bootstrap-cache.outputs.cache-hit != 'true'\n run: |\n edb server --bootstrap-only\n<%- endmacro %>\n\n<% macro calc_cache_key() -%>\n mkdir -p shared-artifacts\n if [ \"$(uname)\" = \"Darwin\" ]; then\n find /usr/lib -type f -name 'lib*' -exec stat -f '%N %z' {} + | sort | shasum -a 256 | cut -d ' ' -f1 > shared-artifacts/lib_cache_key.txt\n else\n find /usr/lib -type f -name 'lib*' -printf '%P %s\\n' | sort | sha256sum | cut -d ' ' -f1 > shared-artifacts/lib_cache_key.txt\n fi\n python setup.py -q ci_helper --type rust >shared-artifacts/rust_cache_key.txt\n python setup.py -q ci_helper --type ext >shared-artifacts/ext_cache_key.txt\n python setup.py -q ci_helper --type parsers >shared-artifacts/parsers_cache_key.txt\n python setup.py -q ci_helper --type postgres >shared-artifacts/postgres_git_rev.txt\n python setup.py -q ci_helper --type libpg_query >shared-artifacts/libpg_query_git_rev.txt\n echo 'f8cd94309eaccbfba5dea7835b88c78377608a37' >shared-artifacts/stolon_git_rev.txt\n python setup.py -q ci_helper --type bootstrap >shared-artifacts/bootstrap_cache_key.txt\n echo POSTGRES_GIT_REV=$(cat shared-artifacts/postgres_git_rev.txt) >> $GITHUB_ENV\n echo LIBPG_QUERY_GIT_REV=$(cat shared-artifacts/libpg_query_git_rev.txt) >> $GITHUB_ENV\n echo STOLON_GIT_REV=$(cat shared-artifacts/stolon_git_rev.txt) >> $GITHUB_ENV\n echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV\n echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV\n<%- endmacro %>\n\n<% macro install_python_requirements() %>\n - name: Download requirements.txt\n uses: actions/cache@v4\n with:\n path: requirements.txt\n key: edb-requirements-${{ hashFiles('pyproject.toml') }}\n\n - name: Install Python dependencies\n run: |\n python -m pip install uv~=0.1.0 && uv pip install -U -r requirements.txt\n # 80.9.0 breaks our sphinx, and it keeps sneaking in\n uv pip install setuptools==80.8.0\n\n<%- endmacro %>\n\n<% macro restore_cache(ref=\"\") %>\n << init(ref) >>\n << install_python_requirements() >>\n\n # Restore the artifacts and environment variables\n\n - name: Download shared artifacts\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: shared-artifacts\n path: shared-artifacts\n\n - name: Set environment variables\n run: |\n echo POSTGRES_GIT_REV=$(cat shared-artifacts/postgres_git_rev.txt) >> $GITHUB_ENV\n echo STOLON_GIT_REV=$(cat shared-artifacts/stolon_git_rev.txt) >> $GITHUB_ENV\n echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV\n echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV\n\n # Restore build cache\n\n - name: Restore cached Rust extensions\n uses: actions/cache@v4\n id: rust-cache\n with:\n path: build/rust_extensions\n key: edb-rust-v4-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n\n - name: Restore cached Cython extensions\n uses: actions/cache@v4\n id: ext-cache\n with:\n path: build/extensions\n key: edb-ext-v6-${{ hashFiles('shared-artifacts/ext_cache_key.txt') }}\n\n - name: Restore compiled parsers cache\n uses: actions/cache@v4\n id: parsers-cache\n with:\n path: build/lib\n key: edb-parsers-v3-${{ hashFiles('shared-artifacts/parsers_cache_key.txt') }}\n\n - name: Restore cached PostgreSQL build\n uses: actions/cache@v4\n id: postgres-cache\n with:\n path: build/postgres/install\n key: edb-postgres-v3-${{ env.POSTGRES_GIT_REV }}-${{ hashFiles('shared-artifacts/lib_cache_key.txt') }}\n\n - name: Restore cached Stolon build\n uses: actions/cache@v4\n id: stolon-cache\n with:\n path: build/stolon/bin\n key: edb-stolon-v2-${{ env.STOLON_GIT_REV }}\n\n - name: Restore bootstrap cache\n uses: actions/cache@v4\n id: bootstrap-cache\n with:\n path: build/cache\n key: edb-bootstrap-v2-${{ hashFiles('shared-artifacts/bootstrap_cache_key.txt') }}\n\n - name: Stop if we cannot retrieve the cache\n if: |\n steps.rust-cache.outputs.cache-hit != 'true' ||\n steps.ext-cache.outputs.cache-hit != 'true' ||\n steps.parsers-cache.outputs.cache-hit != 'true' ||\n steps.postgres-cache.outputs.cache-hit != 'true' ||\n steps.stolon-cache.outputs.cache-hit != 'true' ||\n steps.bootstrap-cache.outputs.cache-hit != 'true'\n run: |\n echo ::error::Cannot retrieve build cache.\n exit 1\n\n - name: Validate cached binaries\n run: |\n # Validate Stolon\n ./build/stolon/bin/stolon-sentinel --version || exit 1\n ./build/stolon/bin/stolon-keeper --version || exit 1\n ./build/stolon/bin/stolon-proxy --version || exit 1\n\n # Validate PostgreSQL\n ./build/postgres/install/bin/postgres --version || exit 1\n ./build/postgres/install/bin/pg_config --version || exit 1\n\n - name: Restore cache into the source tree\n run: |\n rsync -av ./build/rust_extensions/edb/ ./edb/\n rsync -av ./build/extensions/edb/ ./edb/\n rsync -av ./build/lib/edb/ ./edb/\n cp build/postgres/install/stamp build/postgres/\n\n - name: Install edgedb-server\n env:\n BUILD_EXT_MODE: skip\n run: |\n # --no-build-isolation because we have explicitly installed all deps\n # and don't want them to be reinstalled in an \"isolated env\".\n pip install --no-build-isolation --no-deps -e .[test,docs]\n<%- endmacro %>\n\n<% macro setup_terraform() -%>\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n\n - name: Setup Terraform\n uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2.0.3\n\n - name: Initialize Terraform\n run: terraform init\n<%- endmacro %>\n" }, { "path": ".github/workflows.src/tests.inplace.targets.yml", "content": "data:\n" }, { "path": ".github/workflows.src/tests.inplace.tpl.yml", "content": "<% from \"tests.inc.yml\" import build, calc_cache_key, restore_cache -%>\n\nname: Tests of in-place upgrades and patching\n\non:\n schedule:\n - cron: \"0 3 * * *\"\n workflow_dispatch:\n inputs: {}\n push:\n branches:\n - \"A-inplace*\"\n\njobs:\n build:\n runs-on: ubuntu-latest\n\n steps:\n <%- call build() -%>\n - name: Compute cache keys\n run: |\n << calc_cache_key()|indent >>\n <%- endcall %>\n\n test-inplace:\n runs-on: ubuntu-latest\n needs: build\n strategy:\n fail-fast: false\n matrix:\n include:\n - flags:\n tests:\n - flags: --rollback-and-test\n tests:\n # Do the reapply test on a smaller selection of tests, since\n # it is slower.\n - flags: --rollback-and-reapply\n tests: -k test_link_on_target_delete -k test_edgeql_select -k test_dump\n\n steps:\n <<- restore_cache() >>\n\n # Run the test\n # TODO: Would it be better to split this up into multiple jobs?\n - name: Test performing in-place upgrades\n run: |\n ./tests/inplace-testing/test.sh ${{ matrix.flags }} vt ${{ matrix.tests }}\n\n test-patches:\n runs-on: ubuntu-latest\n needs: build\n steps:\n <<- restore_cache() >>\n\n - name: Test performing in-place upgrades\n run: |\n ./tests/patch-testing/test.sh test-dir -k test_link_on_target_delete -k test_edgeql_select -k test_edgeql_scope -k test_dump\n\n compute-versions:\n runs-on: ubuntu-latest\n outputs:\n matrix: ${{ steps.set-matrix.outputs.matrix }}\n steps:\n - uses: actions/checkout@v4\n - id: set-matrix\n name: Compute versions to run on\n run: python3 .github/scripts/patches/compute-ipu-versions.py\n\n\n test:\n runs-on: ubuntu-latest\n needs: [build, compute-versions]\n strategy:\n fail-fast: false\n matrix: ${{fromJSON(needs.compute-versions.outputs.matrix)}}\n\n steps:\n <<- restore_cache() >>\n\n # Run the test\n\n - name: Download an earlier database version\n run: |\n wget -q \"${{ matrix.edgedb-url }}\"\n tar xzf ${{ matrix.edgedb-basename }}-${{ matrix.edgedb-version }}.tar.gz\n\n - name: Make sure a CLI named \"edgedb\" exists (sigh)\n run: |\n ln -s gel $(dirname $(which gel))/edgedb\n\n - name: Test inplace upgrades from previous major version\n run: |\n ./tests/inplace-testing/test-old.sh vt ${{ matrix.edgedb-basename }}-${{ matrix.edgedb-version }}\n\n\n workflow-notifications:\n if: failure() && github.event_name != 'pull_request'\n name: Notify in Slack on failures\n needs:\n - build\n - test-inplace\n - test-patches\n runs-on: ubuntu-latest\n permissions:\n actions: 'read'\n steps:\n - name: Slack Workflow Notification\n uses: Gamesight/slack-workflow-status@26a36836c887f260477432e4314ec3490a84f309\n with:\n repo_token: ${{secrets.GITHUB_TOKEN}}\n slack_webhook_url: ${{secrets.ACTIONS_SLACK_WEBHOOK_URL}}\n name: 'Workflow notifications'\n icon_emoji: ':hammer:'\n include_jobs: 'on-failure'\n" }, { "path": ".github/workflows.src/tests.inplace7x.targets.yml", "content": "data:\n" }, { "path": ".github/workflows.src/tests.inplace7x.tpl.yml", "content": "<% from \"tests.inc.yml\" import build, calc_cache_key, restore_cache -%>\n\nname: Tests of in-place upgrades to 7.x\n\non:\n schedule:\n - cron: \"0 3 * * *\"\n workflow_dispatch:\n inputs: {}\n push:\n branches:\n - \"A-inplace*\"\n\njobs:\n build:\n runs-on: ubuntu-latest\n\n steps:\n <%- call build(\"release/7.x\") -%>\n - name: Compute cache keys\n run: |\n << calc_cache_key()|indent >>\n <%- endcall %>\n\n test-inplace:\n runs-on: ubuntu-latest\n needs: build\n strategy:\n fail-fast: false\n matrix:\n include:\n - flags:\n tests:\n - flags: --rollback-and-test\n tests:\n # Do the reapply test on a smaller selection of tests, since\n # it is slower.\n - flags: --rollback-and-reapply\n tests: -k test_link_on_target_delete -k test_edgeql_select -k test_dump\n\n steps:\n <<- restore_cache(\"release/7.x\") >>\n\n # Run the test\n # TODO: Would it be better to split this up into multiple jobs?\n - name: Test performing in-place upgrades\n run: |\n ./tests/inplace-testing/test.sh ${{ matrix.flags }} vt ${{ matrix.tests }}\n\n compute-versions:\n runs-on: ubuntu-latest\n outputs:\n matrix: ${{ steps.set-matrix.outputs.matrix }}\n steps:\n - uses: actions/checkout@v4\n with:\n fetch-depth: 0\n submodules: false\n ref: release/7.x\n - id: set-matrix\n name: Compute versions to run on\n run: python3 .github/scripts/patches/compute-ipu-versions.py\n\n\n test:\n runs-on: ubuntu-latest\n needs: [build, compute-versions]\n strategy:\n fail-fast: false\n matrix: ${{fromJSON(needs.compute-versions.outputs.matrix)}}\n\n steps:\n <<- restore_cache(\"release/7.x\") >>\n\n # Run the test\n\n - name: Download an earlier database version\n run: |\n wget -q \"${{ matrix.edgedb-url }}\"\n tar xzf ${{ matrix.edgedb-basename }}-${{ matrix.edgedb-version }}.tar.gz\n\n - name: Make sure a CLI named \"edgedb\" exists (sigh)\n run: |\n ln -s gel $(dirname $(which gel))/edgedb\n\n - name: Test inplace upgrades from previous major version\n run: |\n ./tests/inplace-testing/test-old.sh vt ${{ matrix.edgedb-basename }}-${{ matrix.edgedb-version }}\n\n\n workflow-notifications:\n if: failure() && github.event_name != 'pull_request'\n name: Notify in Slack on failures\n needs:\n - build\n - test-inplace\n runs-on: ubuntu-latest\n permissions:\n actions: 'read'\n steps:\n - name: Slack Workflow Notification\n uses: Gamesight/slack-workflow-status@26a36836c887f260477432e4314ec3490a84f309\n with:\n repo_token: ${{secrets.GITHUB_TOKEN}}\n slack_webhook_url: ${{secrets.ACTIONS_SLACK_WEBHOOK_URL}}\n name: 'Workflow notifications'\n icon_emoji: ':hammer:'\n include_jobs: 'on-failure'\n" }, { "path": ".github/workflows.src/tests.managed-pg.targets.yml", "content": "data:\n" }, { "path": ".github/workflows.src/tests.managed-pg.tpl.yml", "content": "<% from \"tests.inc.yml\" import build, calc_cache_key, restore_cache, setup_terraform -%>\n\n<% macro setup_aws_creds() -%>\n - name: Configure AWS Credentials\n uses: aws-actions/configure-aws-credentials@v1\n with:\n aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}\n aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}\n aws-region: us-east-2\n<%- endmacro -%>\n\n<% macro setup_gcp_creds() -%>\n - name: Configure GCP Credentials\n uses: google-github-actions/setup-gcloud@main\n with:\n service_account_key: ${{ secrets.GCP_SA_KEY }}\n export_default_credentials: true\n<%- endmacro -%>\n\nname: Tests on Managed PostgreSQL\n\non:\n schedule:\n - cron: \"0 3 * * 6\"\n workflow_dispatch:\n inputs: {}\n push:\n branches:\n - cloud-test\n\njobs:\n build:\n runs-on: ubuntu-latest\n\n steps:\n <%- call build() -%>\n - name: Compute cache keys\n run: |\n << calc_cache_key()|indent >>\n <%- endcall %>\n\n\n setup-aws-rds:\n runs-on: ubuntu-latest\n outputs:\n pghost: ${{ steps.pghost.outputs.stdout }}\n defaults:\n run:\n working-directory: .github/aws-rds\n steps:\n << setup_terraform()|indent(2) >>\n\n << setup_aws_creds()|indent(2) >>\n\n - name: Setup AWS RDS\n env:\n TF_VAR_sg_id: ${{ secrets.AWS_SECURITY_GROUP }}\n TF_VAR_password: ${{ secrets.AWS_RDS_PASSWORD }}\n run: |\n terraform apply -auto-approve\n\n - name: Store Terraform state\n if: ${{ always() }}\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: aws-rds-tfstate\n path: .github/aws-rds/terraform.tfstate\n retention-days: 1\n\n - name: Get RDS host\n id: pghost\n run: |\n terraform output -raw db_instance_address\n\n test-aws-rds:\n runs-on: ubuntu-latest\n needs: [setup-aws-rds, build]\n steps:\n <<- restore_cache() >>\n\n # Run the test\n\n - name: Test\n env:\n EDGEDB_TEST_BACKEND_DSN: postgres://edbtest:${{ secrets.AWS_RDS_PASSWORD }}@${{ needs.setup-aws-rds.outputs.pghost }}/postgres\n run: |\n edb server --bootstrap-only --backend-dsn=$EDGEDB_TEST_BACKEND_DSN --testmode\n edb test -j2 -v --backend-dsn=$EDGEDB_TEST_BACKEND_DSN\n\n teardown-aws-rds:\n runs-on: ubuntu-latest\n needs: test-aws-rds\n if: ${{ always() }}\n defaults:\n run:\n working-directory: .github/aws-rds\n steps:\n << setup_terraform()|indent(2) >>\n\n << setup_aws_creds()|indent(2) >>\n\n - name: Restore Terraform state\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: aws-rds-tfstate\n path: .github/aws-rds\n\n - name: Destroy AWS RDS\n run: terraform destroy -auto-approve\n env:\n TF_VAR_sg_id: ${{ secrets.AWS_SECURITY_GROUP }}\n TF_VAR_password: ${{ secrets.AWS_RDS_PASSWORD }}\n\n - name: Overwrite Terraform state\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: aws-rds-tfstate\n path: .github/aws-rds/terraform.tfstate\n retention-days: 1\n\n\n setup-do-database:\n runs-on: ubuntu-latest\n defaults:\n run:\n working-directory: .github/do-database\n steps:\n << setup_terraform()|indent(2) >>\n\n - name: Setup DigitalOcean Database\n env:\n TF_VAR_do_token: ${{ secrets.DIGITALOCEAN_TOKEN }}\n run: |\n terraform apply -auto-approve\n\n - name: Store Terraform state\n if: ${{ always() }}\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: do-database-tfstate\n path: .github/do-database/terraform.tfstate\n retention-days: 1\n\n test-do-database:\n runs-on: ubuntu-latest\n needs: [setup-do-database, build]\n steps:\n <<- restore_cache() >>\n\n - name: Setup Terraform\n uses: hashicorp/setup-terraform@v1\n\n - name: Initialize Terraform\n working-directory: .github/do-database\n run: terraform init\n\n - name: Restore Terraform state\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: do-database-tfstate\n path: .github/do-database\n\n - name: Get Database host\n id: pghost\n working-directory: .github/do-database\n run: |\n terraform output -raw db_instance_address\n\n - name: Get Database port\n id: pgport\n working-directory: .github/do-database\n run: |\n terraform output -raw db_instance_port\n\n - name: Get Database user\n id: pguser\n working-directory: .github/do-database\n run: |\n terraform output -raw db_instance_user\n\n - name: Get Database password\n id: pgpass\n working-directory: .github/do-database\n run: |\n terraform output -raw db_instance_password\n\n - name: Get Database dbname\n id: pgdatabase\n working-directory: .github/do-database\n run: |\n terraform output -raw db_instance_database\n\n # Run the test\n\n - name: Test\n env:\n EDGEDB_TEST_BACKEND_DSN: postgres://${{ steps.pguser.outputs.stdout }}:${{ steps.pgpass.outputs.stdout }}@${{ steps.pghost.outputs.stdout }}:${{ steps.pgport.outputs.stdout }}/${{ steps.pgdatabase.outputs.stdout }}\n run: |\n edb server --bootstrap-only --backend-dsn=$EDGEDB_TEST_BACKEND_DSN --testmode\n edb test -j2 -v --backend-dsn=$EDGEDB_TEST_BACKEND_DSN\n\n teardown-do-database:\n runs-on: ubuntu-latest\n needs: test-do-database\n if: ${{ always() }}\n defaults:\n run:\n working-directory: .github/do-database\n steps:\n << setup_terraform()|indent(2) >>\n\n - name: Restore Terraform state\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: do-database-tfstate\n path: .github/do-database\n\n - name: Destroy DigitalOcean Database\n run: terraform destroy -auto-approve\n env:\n TF_VAR_do_token: ${{ secrets.DIGITALOCEAN_TOKEN }}\n\n - name: Overwrite Terraform state\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: do-database-tfstate\n path: .github/do-database/terraform.tfstate\n retention-days: 1\n\n\n setup-gcp-cloud-sql:\n runs-on: ubuntu-latest\n outputs:\n pghost: ${{ steps.pghost.outputs.stdout }}\n defaults:\n run:\n working-directory: .github/gcp-cloud-sql\n steps:\n << setup_terraform()|indent(2) >>\n\n << setup_gcp_creds()|indent(2) >>\n\n - name: Setup GCP Cloud SQL\n env:\n TF_VAR_password: ${{ secrets.AWS_RDS_PASSWORD }}\n run: |\n terraform apply -auto-approve\n\n - name: Store Terraform state\n if: ${{ always() }}\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: gcp-cloud-sql-tfstate\n path: .github/gcp-cloud-sql/terraform.tfstate\n retention-days: 1\n\n - name: Get Cloud SQL host\n id: pghost\n run: |\n terraform output -raw db_instance_address\n\n test-gcp-cloud-sql:\n runs-on: ubuntu-latest\n needs: [setup-gcp-cloud-sql, build]\n steps:\n <<- restore_cache() >>\n\n # Run the test\n\n - name: Test\n env:\n EDGEDB_TEST_BACKEND_DSN: postgres://postgres:${{ secrets.AWS_RDS_PASSWORD }}@${{ needs.setup-gcp-cloud-sql.outputs.pghost }}/postgres\n run: |\n edb server --bootstrap-only --backend-dsn=$EDGEDB_TEST_BACKEND_DSN --testmode\n edb test -j2 -v --backend-dsn=$EDGEDB_TEST_BACKEND_DSN\n\n teardown-gcp-cloud-sql:\n runs-on: ubuntu-latest\n needs: test-gcp-cloud-sql\n if: ${{ always() }}\n defaults:\n run:\n working-directory: .github/gcp-cloud-sql\n steps:\n << setup_terraform()|indent(2) >>\n\n << setup_gcp_creds()|indent(2) >>\n\n - name: Restore Terraform state\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: gcp-cloud-sql-tfstate\n path: .github/gcp-cloud-sql\n\n - name: Destroy GCP Cloud SQL\n run: terraform destroy -auto-approve\n env:\n TF_VAR_password: ${{ secrets.AWS_RDS_PASSWORD }}\n\n - name: Overwrite Terraform state\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: gcp-cloud-sql-tfstate\n path: .github/gcp-cloud-sql/terraform.tfstate\n retention-days: 1\n\n\n setup-aws-aurora:\n runs-on: ubuntu-latest\n outputs:\n pghost: ${{ steps.pghost.outputs.stdout }}\n defaults:\n run:\n working-directory: .github/aws-aurora\n steps:\n << setup_terraform()|indent(2) >>\n\n << setup_aws_creds()|indent(2) >>\n\n - name: Setup AWS RDS Aurora\n env:\n TF_VAR_sg_id: ${{ secrets.AWS_SECURITY_GROUP }}\n TF_VAR_password: ${{ secrets.AWS_RDS_PASSWORD }}\n TF_VAR_vpc_id: ${{ secrets.AWS_VPC_ID }}\n run: |\n terraform apply -auto-approve\n\n - name: Store Terraform state\n if: ${{ always() }}\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: aws-aurora-tfstate\n path: .github/aws-aurora/terraform.tfstate\n retention-days: 1\n\n - name: Get RDS Aurora host\n id: pghost\n run: |\n terraform output -raw rds_cluster_endpoint\n\n test-aws-aurora:\n runs-on: ubuntu-latest\n needs: [setup-aws-aurora, build]\n steps:\n <<- restore_cache() >>\n\n # Run the test\n\n - name: Test\n env:\n EDGEDB_TEST_BACKEND_DSN: postgres://edbtest:${{ secrets.AWS_RDS_PASSWORD }}@${{ needs.setup-aws-aurora.outputs.pghost }}/postgres\n run: |\n edb server --bootstrap-only --backend-dsn=$EDGEDB_TEST_BACKEND_DSN --testmode\n edb test -j1 -v --backend-dsn=$EDGEDB_TEST_BACKEND_DSN\n\n teardown-aws-aurora:\n runs-on: ubuntu-latest\n needs: test-aws-aurora\n if: ${{ always() }}\n defaults:\n run:\n working-directory: .github/aws-aurora\n steps:\n << setup_terraform()|indent(2) >>\n\n << setup_aws_creds()|indent(2) >>\n\n - name: Restore Terraform state\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: aws-aurora-tfstate\n path: .github/aws-aurora\n\n - name: Destroy AWS RDS Aurora\n run: terraform destroy -auto-approve\n env:\n TF_VAR_sg_id: ${{ secrets.AWS_SECURITY_GROUP }}\n TF_VAR_password: ${{ secrets.AWS_RDS_PASSWORD }}\n TF_VAR_vpc_id: ${{ secrets.AWS_VPC_ID }}\n\n - name: Overwrite Terraform state\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: aws-aurora-tfstate\n path: .github/aws-aurora/terraform.tfstate\n retention-days: 1\n\n\n setup-heroku-postgres:\n runs-on: ubuntu-latest\n defaults:\n run:\n working-directory: .github/heroku-postgres\n steps:\n << setup_terraform()|indent(2) >>\n\n - name: Setup Heroku Postgres\n env:\n HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}\n HEROKU_EMAIL: ${{ secrets.HEROKU_EMAIL }}\n run: |\n terraform apply -auto-approve\n\n - name: Store Terraform state\n if: ${{ always() }}\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: heroku-postgres-tfstate\n path: .github/heroku-postgres/terraform.tfstate\n retention-days: 1\n\n test-heroku-postgres:\n runs-on: ubuntu-latest\n needs: [setup-heroku-postgres, build]\n steps:\n <<- restore_cache() >>\n\n - name: Setup Terraform\n uses: hashicorp/setup-terraform@v1\n\n - name: Initialize Terraform\n working-directory: .github/heroku-postgres\n run: terraform init\n\n - name: Restore Terraform state\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: heroku-postgres-tfstate\n path: .github/heroku-postgres\n\n - name: Get Heroku Postgres DSN\n id: pgdsn\n working-directory: .github/heroku-postgres\n run: |\n terraform output -raw heroku_postgres_dsn\n\n # Run the test\n\n - name: Test\n env:\n EDGEDB_TEST_BACKEND_VENDOR: heroku-postgres\n EDGEDB_TEST_BACKEND_DSN: ${{ steps.pgdsn.outputs.stdout }}\n run: |\n edb server --bootstrap-only --backend-dsn=$EDGEDB_TEST_BACKEND_DSN --testmode\n edb test -j1 -v --backend-dsn=$EDGEDB_TEST_BACKEND_DSN\n\n teardown-heroku-postgres:\n runs-on: ubuntu-latest\n needs: test-heroku-postgres\n if: ${{ always() }}\n defaults:\n run:\n working-directory: .github/heroku-postgres\n steps:\n << setup_terraform()|indent(2) >>\n\n << setup_aws_creds()|indent(2) >>\n\n - name: Restore Terraform state\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: heroku-postgres-tfstate\n path: .github/heroku-postgres\n\n - name: Destroy Heroku Postgres\n run: terraform destroy -auto-approve\n env:\n HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}\n HEROKU_EMAIL: ${{ secrets.HEROKU_EMAIL }}\n\n - name: Overwrite Terraform state\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: heroku-postgres-tfstate\n path: .github/heroku-postgres/terraform.tfstate\n retention-days: 1\n\n\n workflow-notifications:\n if: failure() && github.event_name != 'pull_request'\n name: Notify in Slack on failures\n needs:\n - setup-aws-rds\n - test-aws-rds\n - teardown-aws-rds\n - setup-do-database\n - test-do-database\n - teardown-do-database\n - setup-gcp-cloud-sql\n - test-gcp-cloud-sql\n - teardown-gcp-cloud-sql\n - setup-aws-aurora\n - test-aws-aurora\n - teardown-aws-aurora\n - setup-heroku-postgres\n - test-heroku-postgres\n - teardown-heroku-postgres\n runs-on: ubuntu-latest\n permissions:\n actions: 'read'\n steps:\n - name: Slack Workflow Notification\n uses: Gamesight/slack-workflow-status@26a36836c887f260477432e4314ec3490a84f309\n with:\n repo_token: ${{secrets.GITHUB_TOKEN}}\n slack_webhook_url: ${{secrets.ACTIONS_SLACK_WEBHOOK_URL}}\n name: 'Workflow notifications'\n icon_emoji: ':hammer:'\n include_jobs: 'on-failure'\n" }, { "path": ".github/workflows.src/tests.patches.targets.yml", "content": "data:\n" }, { "path": ".github/workflows.src/tests.patches.tpl.yml", "content": "<% from \"tests.inc.yml\" import build, calc_cache_key, restore_cache -%>\n\nname: Tests of patching old EdgeDB Versions\n\non:\n workflow_dispatch:\n inputs: {}\n pull_request:\n branches:\n - release/*\n push:\n branches:\n - patch-test*\n - release/*\n\njobs:\n build:\n runs-on: ubuntu-latest\n\n steps:\n <%- call build() -%>\n - name: Compute cache keys\n run: |\n << calc_cache_key()|indent >>\n <%- endcall %>\n\n compute-versions:\n runs-on: ubuntu-latest\n outputs:\n matrix: ${{ steps.set-matrix.outputs.matrix }}\n steps:\n - uses: actions/checkout@v4\n - id: set-matrix\n name: Compute versions to run on\n run: python3 .github/scripts/patches/compute-versions.py\n\n test:\n runs-on: ubuntu-latest\n needs: [build, compute-versions]\n strategy:\n fail-fast: false\n matrix: ${{fromJSON(needs.compute-versions.outputs.matrix)}}\n\n steps:\n <<- restore_cache() >>\n\n # Run the test\n\n - name: Download an earlier database version and set up a instance\n run: |\n wget -q \"${{ matrix.edgedb-url }}\"\n tar xzf ${{ matrix.edgedb-basename }}-${{ matrix.edgedb-version }}.tar.gz\n ${{ matrix.edgedb-basename }}-${{ matrix.edgedb-version }}/bin/edgedb-server -D test-dir --bootstrap-only --testmode\n\n - name: Create databases on the older version\n if: ${{ matrix.make-dbs }}\n run: python3 .github/scripts/patches/create-databases.py ${{ matrix.edgedb-basename }}-${{ matrix.edgedb-version }}/bin/edgedb-server\n\n - name: Run tests with instance created on an older version\n run: |\n # Run the server explicitly first to do the upgrade, since edb test\n # has timeouts.\n edb server --bootstrap-only --data-dir test-dir\n # Should we run *all* the tests?\n edb test -j2 -v --data-dir test-dir tests/test_edgeql_json.py tests/test_edgeql_casts.py tests/test_edgeql_functions.py tests/test_edgeql_expressions.py tests/test_edgeql_policies.py tests/test_edgeql_vector.py tests/test_edgeql_scope.py tests/test_http_ext_auth.py\n\n - name: Test downgrading a database after an upgrade\n if: ${{ !contains(matrix.edgedb-version, '-rc') && !contains(matrix.edgedb-version, '-beta') }}\n env:\n EDGEDB_VERSION: ${{ matrix.edgedb-version }}\n run: python3 .github/scripts/patches/test-downgrade.py\n\n workflow-notifications:\n if: failure() && github.event_name != 'pull_request'\n name: Notify in Slack on failures\n needs:\n - build\n - compute-versions\n - test\n runs-on: ubuntu-latest\n permissions:\n actions: 'read'\n steps:\n - name: Slack Workflow Notification\n uses: Gamesight/slack-workflow-status@26a36836c887f260477432e4314ec3490a84f309\n with:\n repo_token: ${{secrets.GITHUB_TOKEN}}\n slack_webhook_url: ${{secrets.ACTIONS_SLACK_WEBHOOK_URL}}\n name: 'Workflow notifications'\n icon_emoji: ':hammer:'\n include_jobs: 'on-failure'\n" }, { "path": ".github/workflows.src/tests.pg-versions.targets.yml", "content": "data:\n" }, { "path": ".github/workflows.src/tests.pg-versions.tpl.yml", "content": "<% from \"tests.inc.yml\" import build, calc_cache_key, restore_cache, setup_terraform -%>\n\nname: Tests on PostgreSQL Versions\n\non:\n schedule:\n - cron: \"0 3 * * *\"\n workflow_dispatch:\n inputs: {}\n push:\n branches:\n - pg-test\n\njobs:\n build:\n runs-on: ubuntu-latest\n\n steps:\n <%- call build() -%>\n - name: Compute cache keys\n run: |\n << calc_cache_key()|indent >>\n <%- endcall %>\n\n test:\n runs-on: ubuntu-latest\n needs: build\n strategy:\n fail-fast: false\n matrix:\n postgres-version: [ 17 ]\n single-mode:\n - ''\n # These are very broken. Disabling them for now until we\n # decide whether to fix them or give up.\n # - 'NOCREATEDB NOCREATEROLE'\n # - 'CREATEDB NOCREATEROLE'\n multi-tenant-mode: [ '' ]\n include:\n - postgres-version: 14\n single-mode: ''\n multi-tenant-mode: ''\n - postgres-version: 15\n single-mode: ''\n multi-tenant-mode: ''\n - postgres-version: 16\n single-mode: ''\n multi-tenant-mode: ''\n - postgres-version: 17\n single-mode: ''\n multi-tenant-mode: 'remote-compiler'\n - postgres-version: 17\n single-mode: ''\n multi-tenant-mode: 'multi-tenant'\n services:\n postgres:\n image: pgvector/pgvector:0.7.4-pg${{ matrix.postgres-version }}\n env:\n POSTGRES_PASSWORD: postgres\n options: >-\n --health-cmd pg_isready\n --health-interval 10s\n --health-timeout 5s\n --health-retries 5\n --name postgres\n ports:\n - 5432:5432\n\n steps:\n - name: Trust pgvector extension\n uses: docker://docker\n with:\n args: docker exec postgres sed -i $a\\trusted=true /usr/share/postgresql/${{ matrix.postgres-version }}/extension/vector.control\n\n <<- restore_cache() >>\n\n # Run the test\n\n - name: Setup single mode role and database\n if: ${{ matrix.single-mode }}\n shell: python\n run: |\n import asyncio\n import subprocess\n\n from edb.server.pgcluster import get_pg_bin_dir\n\n async def main():\n psql = await get_pg_bin_dir() / \"psql\"\n dsn = \"postgres://postgres:postgres@localhost/postgres\"\n\n script = \"\"\"\\\n CREATE ROLE singles;\n ALTER ROLE singles WITH LOGIN PASSWORD 'test' NOSUPERUSER\n ${{ matrix.single-mode }};\n CREATE DATABASE singles OWNER singles;\n REVOKE ALL ON DATABASE singles FROM PUBLIC;\n GRANT CONNECT ON DATABASE singles TO singles;\n GRANT ALL ON DATABASE singles TO singles;\n \"\"\"\n\n subprocess.run(\n [str(psql), dsn],\n check=True,\n text=True,\n input=script,\n )\n\n asyncio.run(main())\n\n - name: Test\n env:\n EDGEDB_TEST_POSTGRES_VERSION: ${{ matrix.postgres-version }}\n run: |\n if [[ \"${{ matrix.single-mode }}\" ]]; then\n export EDGEDB_TEST_BACKEND_DSN=postgres://singles:test@localhost/singles\n else\n export EDGEDB_TEST_BACKEND_DSN=postgres://postgres:postgres@localhost/postgres\n fi\n if [[ \"${{ matrix.multi-tenant-mode }}\" == \"remote-compiler\" ]]; then\n export EDGEDB_TEST_REMOTE_COMPILER=localhost:5660\n export _EDGEDB_SERVER_COMPILER_POOL_SECRET=secret\n __EDGEDB_DEVMODE=1 edgedb-server compiler --pool-size 2 &\n fi\n edb server --bootstrap-only --backend-dsn=$EDGEDB_TEST_BACKEND_DSN --testmode\n if [[ \"${{ matrix.multi-tenant-mode }}\" == \"multi-tenant\" ]]; then\n export EDGEDB_SERVER_MULTITENANT_CONFIG_FILE=/tmp/edb.mt.json\n echo \"{\\\"localhost\\\":{\\\"instance-name\\\":\\\"localtest\\\",\\\"backend-dsn\\\":\\\"$EDGEDB_TEST_BACKEND_DSN\\\",\\\"admin\\\":true,\\\"max-backend-connections\\\":10}}\" > /tmp/edb.mt.json\n fi\n if [[ \"${{ matrix.single-mode }}\" == *\"NOCREATEDB\"* ]]; then\n edb test -j1 -v --backend-dsn=$EDGEDB_TEST_BACKEND_DSN\n else\n edb test -j2 -v --backend-dsn=$EDGEDB_TEST_BACKEND_DSN\n fi\n\n\n workflow-notifications:\n if: failure() && github.event_name != 'pull_request'\n name: Notify in Slack on failures\n needs:\n - build\n - test\n runs-on: ubuntu-latest\n permissions:\n actions: 'read'\n steps:\n - name: Slack Workflow Notification\n uses: Gamesight/slack-workflow-status@26a36836c887f260477432e4314ec3490a84f309\n with:\n repo_token: ${{secrets.GITHUB_TOKEN}}\n slack_webhook_url: ${{secrets.ACTIONS_SLACK_WEBHOOK_URL}}\n name: 'Workflow notifications'\n icon_emoji: ':hammer:'\n include_jobs: 'on-failure'\n" }, { "path": ".github/workflows.src/tests.pool.targets.yml", "content": "data:\n" }, { "path": ".github/workflows.src/tests.pool.tpl.yml", "content": "<% from \"tests.inc.yml\" import build, calc_cache_key -%>\n\nname: Pool Simulation Test\n\non:\n push:\n branches:\n - master\n - pool-test\n paths:\n - 'edb/server/connpool/**'\n - 'edb/server/conn_pool/**'\n - 'tests/test_server_pool.py'\n - '.github/workflows/tests-pool.yml'\n pull_request:\n branches:\n - master\n paths:\n - 'edb/server/connpool/**'\n - 'edb/server/conn_pool/**'\n - 'tests/test_server_pool.py'\n - '.github/workflows/tests-pool.yml'\n\njobs:\n test:\n runs-on: ubuntu-latest\n concurrency: pool-test\n steps:\n <%- call build() -%>\n - name: Compute cache keys\n run: |\n << calc_cache_key()|indent >>\n <%- endcall %>\n\n - uses: actions/checkout@v4\n if: startsWith(github.ref, 'refs/heads')\n with:\n repository: edgedb/edgedb-pool-simulation\n path: pool-simulation\n token: ${{ secrets.GITHUB_CI_BOT_TOKEN }}\n\n - name: Run the pool simulation test\n env:\n PYTHONPATH: .\n SIMULATION_CI: yes\n TIME_SCALE: 10\n run: |\n mkdir -p pool-simulation/reports\n python tests/test_server_pool.py\n\n - uses: EndBug/add-and-commit@v7.0.0\n if: ${{ always() }}\n continue-on-error: true\n with:\n branch: main\n cwd: pool-simulation\n author_name: github-actions\n author_email: 41898282+github-actions[bot]@users.noreply.github.com\n" }, { "path": ".github/workflows.src/tests.reflection.targets.yml", "content": "data:\n" }, { "path": ".github/workflows.src/tests.reflection.tpl.yml", "content": "<% from \"tests.inc.yml\" import build, calc_cache_key, restore_cache -%>\nname: Tests with reflection validation\n\non:\n schedule:\n - cron: \"0 3 * * *\"\n workflow_dispatch:\n inputs: {}\n push:\n branches:\n - \"REFL-*\"\n\njobs:\n build:\n runs-on: ubuntu-latest\n\n steps:\n <%- call build() -%>\n\n - name: Compute cache keys\n env:\n GIST_TOKEN: ${{ secrets.CI_BOT_GIST_TOKEN }}\n run: |\n << calc_cache_key()|indent >>\n <%- endcall %>\n\n test:\n needs: build\n runs-on: ubuntu-latest\n\n steps:\n <<- restore_cache() >>\n\n # Run the test\n\n - name: Test\n env:\n EDGEDB_DEBUG_DELTA_VALIDATE_REFLECTION: 1\n run: |\n edb test -j2 -v\n\n workflow-notifications:\n if: failure() && github.event_name != 'pull_request'\n name: Notify in Slack on failures\n needs:\n - build\n - test\n runs-on: ubuntu-latest\n permissions:\n actions: 'read'\n steps:\n - name: Slack Workflow Notification\n uses: Gamesight/slack-workflow-status@26a36836c887f260477432e4314ec3490a84f309\n with:\n repo_token: ${{secrets.GITHUB_TOKEN}}\n slack_webhook_url: ${{secrets.ACTIONS_SLACK_WEBHOOK_URL}}\n name: 'Workflow notifications'\n icon_emoji: ':hammer:'\n include_jobs: 'on-failure'\n" }, { "path": ".github/workflows.src/tests.targets.yml", "content": "data:\n" }, { "path": ".github/workflows.src/tests.tpl.yml", "content": "<% from \"tests.inc.yml\" import init, build, calc_cache_key, install_python_requirements, restore_cache -%>\nname: Tests\n\non:\n push:\n branches:\n - master\n - ci\n - \"release/*\"\n pull_request:\n branches:\n - '**'\n schedule:\n - cron: \"0 */3 * * *\"\n\njobs:\n build:\n runs-on: ubuntu-latest\n\n steps:\n <%- call build() -%>\n - name: Compute cache keys and download the running times log\n env:\n GIST_TOKEN: ${{ secrets.CI_BOT_GIST_TOKEN }}\n run: |\n << calc_cache_key()|indent >>\n\n curl \\\n -H \"Accept: application/vnd.github.v3+json\" \\\n -u edgedb-ci:$GIST_TOKEN \\\n https://api.github.com/gists/8b722a65397f7c4c0df72f5394efa04c \\\n | jq '.files.\"time_stats.csv\".raw_url' \\\n | xargs curl > shared-artifacts/time_stats.csv\n <%- endcall %>\n\n cargo-test:\n needs: build\n runs-on: ubuntu-latest\n\n steps:\n << init() >>\n << install_python_requirements() >>\n\n - name: Download cache key\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: shared-artifacts\n path: shared-artifacts\n\n - name: Generate environment variables\n run: |\n echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV\n\n - name: Handle Rust extensions build cache\n uses: actions/cache@v4\n id: rust-cache\n with:\n path: ${{ env.BUILD_TEMP }}/rust/extensions\n key: edb-rust-build-v1-${{ hashFiles('shared-artifacts/rust_cache_key.txt') }}\n\n - name: Install Rust toolchain\n uses: dsherret/rust-toolchain-file@v1\n\n - name: Cargo test\n env:\n CARGO_TARGET_DIR: ${{ env.BUILD_TEMP }}/rust/extensions\n CARGO_HOME: ${{ env.BUILD_TEMP }}/rust/extensions/cargo_home\n run:\n cargo test --all-features\n\n python-test:\n needs: build\n runs-on: ubuntu-latest\n strategy:\n fail-fast: false\n matrix:\n shard: [\n 1, 2, 3, 4,\n 5, 6, 7, 8,\n 9, 10, 11, 12,\n 13, 14, 15, 16,\n ]\n\n steps:\n <<- restore_cache() >>\n\n # Run the test\n\n - name: Install Rust toolchain\n uses: dsherret/rust-toolchain-file@v1\n\n - name: Test\n env:\n SHARD: ${{ matrix.shard }}\n EDGEDB_TEST_REPEATS: 1\n run: |\n mkdir -p results/\n cp shared-artifacts/time_stats.csv results/running_times_${SHARD}.csv\n edb test --jobs 2 --verbose --shard ${SHARD}/16 \\\n --running-times-log=results/running_times_${SHARD}.csv \\\n --result-log=results/result_${SHARD}.json\n\n - name: Upload test results\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n if: ${{ always() }}\n with:\n name: python-test-results-${{ matrix.shard }}\n path: results\n retention-days: 1\n\n python-test-list:\n needs: build\n runs-on: ubuntu-latest\n steps:\n <<- restore_cache() >>\n\n # List tests and upload\n\n - name: Generate complete list of tests for verification\n env:\n SHARD: ${{ matrix.shard }}\n EDGEDB_TEST_REPEATS: 1\n run: |\n edb test --list > shared-artifacts/all_tests.txt\n\n - name: Upload list of tests\n uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0\n with:\n name: test-list\n path: shared-artifacts\n retention-days: 1\n\n test-conclusion:\n needs: [cargo-test, python-test, python-test-list]\n runs-on: ubuntu-latest\n if: ${{ always() }}\n steps:\n - name: Set up Python\n uses: actions/setup-python@v5\n with:\n python-version: '3.12.2'\n\n - name: Install Python deps\n run: |\n python -m pip install requests click\n\n - uses: actions/checkout@v4\n with:\n submodules: false\n\n - name: Download python-test results\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n pattern: python-test-results-*\n merge-multiple: true\n path: results\n\n # Render results and exit if they were unsuccessful\n - name: Render results\n run: |\n python edb/tools/test/results.py 'results/result_*.json'\n\n - name: Download shared artifacts\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: shared-artifacts\n path: shared-artifacts\n\n - name: Download test list\n uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8\n with:\n name: test-list\n path: shared-artifacts\n\n - name: Merge stats and verify tests completion\n shell: python\n env:\n GIST_TOKEN: ${{ secrets.CI_BOT_GIST_TOKEN }}\n GIT_REF: ${{ github.ref }}\n run: |\n import csv\n import glob\n import io\n import os\n import requests\n\n orig = {}\n new = {}\n all_tests = set()\n with open(\"shared-artifacts/time_stats.csv\") as f:\n for name, t, c in csv.reader(f):\n assert name not in orig, \"duplicate test name in original stats!\"\n orig[name] = (t, int(c))\n\n with open(\"shared-artifacts/all_tests.txt\") as f:\n for line in f:\n assert line not in all_tests, \"duplicate test name in this run!\"\n all_tests.add(line.strip())\n\n for new_file in glob.glob(\"results/running_times_*.csv\"):\n with open(new_file) as f:\n for name, t, c in csv.reader(f):\n if int(c) > orig.get(name, (0, 0))[1]:\n if name.startswith(\"setup::\"):\n new[name] = (t, c)\n else:\n assert name not in new, f\"duplicate test! {name}\"\n new[name] = (t, c)\n all_tests.remove(name)\n\n assert not all_tests, \"Tests not run! \\n\" + \"\\n\".join(all_tests)\n\n if os.environ[\"GIT_REF\"] == \"refs/heads/master\":\n buf = io.StringIO()\n writer = csv.writer(buf)\n orig.update(new)\n for k, v in sorted(orig.items()):\n writer.writerow((k,) + v)\n\n resp = requests.patch(\n \"https://api.github.com/gists/8b722a65397f7c4c0df72f5394efa04c\",\n headers={\"Accept\": \"application/vnd.github.v3+json\"},\n auth=(\"edgedb-ci\", os.environ[\"GIST_TOKEN\"]),\n json={\"files\": {\"time_stats.csv\": {\"content\": buf.getvalue()}}},\n )\n resp.raise_for_status()\n\n\n workflow-notifications:\n if: failure() && github.event_name != 'pull_request'\n name: Notify in Slack on failures\n needs:\n - test-conclusion\n runs-on: ubuntu-latest\n permissions:\n actions: 'read'\n steps:\n - name: Slack Workflow Notification\n uses: Gamesight/slack-workflow-status@26a36836c887f260477432e4314ec3490a84f309\n with:\n repo_token: ${{secrets.GITHUB_TOKEN}}\n slack_webhook_url: ${{secrets.ACTIONS_SLACK_WEBHOOK_URL}}\n name: 'Workflow notifications'\n icon_emoji: ':hammer:'\n include_jobs: 'on-failure'\n" }, { "path": ".gitignore", "content": "*._*\n*.pyc\n*.pyo\n*.o\n*.so\n*.dylib\n.vscode/\n.zed/\n.helix/\n*~\n.#*\n.*.swp\n.DS_Store\n\\#*#\n/test*.py\n/.local\n/perf.data*\n/build\n/target\n/tmp\n__pycache__/\n.d8_history\n/.venv\n/.eggs\n/*.egg\n/*.egg-info\n/dist\n/.cache\ndocs/_build\n/AUTHORS\n/ChangeLog\n/tests/dumps/**/*_dev*.dump\n/edb/_buildmeta.py\n\n/.coverage*\n/htmlcov\n\n*.code-workspace\n/.pytest_cache\n/.mypy_cache\n/.vagga\n/.dmypy.json\n/compile_commands.json\n/pyrightconfig.json\n" }, { "path": ".gitmodules", "content": "[submodule \"postgres\"]\n\tpath = postgres\n\turl = https://github.com/geldata/postgres.git\n\tignore = untracked\n[submodule \"edb/server/pgproto\"]\n\tpath = edb/server/pgproto\n\turl = https://github.com/MagicStack/py-pgproto.git\n[submodule \"edb/pgsql/parser/libpg_query\"]\n\tpath = edb/pgsql/parser/libpg_query\n\turl = https://github.com/geldata/libpg_query.git\n" }, { "path": ".mailmap", "content": "Elvis Pranskevichus \nYury Selivanov \nYury Selivanov Yuri Selivanov \nVictor Petrovykh \nVictor Petrovykh Vicor Petrovykh \n" }, { "path": "CODE_OF_CONDUCT.md", "content": "# Contributor Covenant Code of Conduct\n\n## Our Pledge\n\nIn the interest of fostering an open and welcoming environment, we as\ncontributors and maintainers pledge to making participation in our project and\nour community a harassment-free experience for everyone, regardless of age, body\nsize, disability, ethnicity, sex characteristics, gender identity and expression,\nlevel of experience, education, socio-economic status, nationality, personal\nappearance, race, religion, or sexual identity and orientation.\n\n## Our Standards\n\nExamples of behavior that contributes to creating a positive environment\ninclude:\n\n* Using welcoming and inclusive language\n* Being respectful of differing viewpoints and experiences\n* Gracefully accepting constructive criticism\n* Focusing on what is best for the community\n* Showing empathy towards other community members\n\nExamples of unacceptable behavior by participants include:\n\n* The use of sexualized language or imagery and unwelcome sexual attention or\n advances\n* Trolling, insulting/derogatory comments, and personal or political attacks\n* Public or private harassment\n* Publishing others' private information, such as a physical or electronic\n address, without explicit permission\n* Other conduct which could reasonably be considered inappropriate in a\n professional setting\n\n## Our Responsibilities\n\nProject maintainers are responsible for clarifying the standards of acceptable\nbehavior and are expected to take appropriate and fair corrective action in\nresponse to any instances of unacceptable behavior.\n\nProject maintainers have the right and responsibility to remove, edit, or\nreject comments, commits, code, wiki edits, issues, and other contributions\nthat are not aligned to this Code of Conduct, or to ban temporarily or\npermanently any contributor for other behaviors that they deem inappropriate,\nthreatening, offensive, or harmful.\n\n## Scope\n\nThis Code of Conduct applies both within project spaces and in public spaces\nwhen an individual is representing the project or its community. Examples of\nrepresenting a project or community include using an official project e-mail\naddress, posting via an official social media account, or acting as an appointed\nrepresentative at an online or offline event. Representation of a project may be\nfurther defined and clarified by project maintainers.\n\n## Enforcement\n\nInstances of abusive, harassing, or otherwise unacceptable behavior may be\nreported by contacting the project team at coc@edgedb.com. All\ncomplaints will be reviewed and investigated and will result in a response that\nis deemed necessary and appropriate to the circumstances. The project team is\nobligated to maintain confidentiality with regard to the reporter of an incident.\nFurther details of specific enforcement policies may be posted separately.\n\nProject maintainers who do not follow or enforce the Code of Conduct in good\nfaith may face temporary or permanent repercussions as determined by other\nmembers of the project's leadership.\n\n## Attribution\n\nThis Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,\navailable at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html\n\n[homepage]: https://www.contributor-covenant.org\n\nFor answers to common questions about this code of conduct, see\nhttps://www.contributor-covenant.org/faq\n" }, { "path": "CONTRIBUTING.rst", "content": "How to contribute to Gel\n========================\n\nThank you for contributing to Gel! We love our open source community and\nwant to foster a healthy contributor ecosystem.\n\nTo make sure the project can continue to improve quickly, we have a few\nguidelines designed to make it easier for your contributions to make it into\nthe project.\n\nThese are guidelines rather than hard rules. If you want to submit a pull\nrequest that strays from these, it might be a good idea to start a discussion\nabout it first. Otherwise, it's possible your pull request might not be merged.\n\nAll contributions\n-----------------\n\n- **Avoid making pull requests that do not have an associated Github Issue.**\n This could be an already existing issue or one you create yourself when you\n discover the problem. This will allow the team to help you scope your\n solution, warn you of potential gotchas, or give you a heads-up on solutions\n that are likely not feasible. It's a good idea to mention in the issue that\n you'd like to contribute code to resolve the issue. **If you're fixing\n something trivial like a typo,** an associated issue isn't necessary.\n- **Write good commit messages.** The subject of your commit message ā€” that's\n the first line ā€” should tell us *what* you did. The body of your message ā€”\n that's the rest of it ā€” should tell us *why* you did it (unless that's\n self-evident).\n\nContributing code\n--------------------------\n\n- **Pull requests without thorough testing are not likely to be merged.** If\n you're not sure if yours is well-tested enough, go ahead and submit. We can\n help guide you to the finish line.\n\nContributing documentation\n--------------------------\n\n- **Avoid changes that don't fix an obvious mistake or add clarity.** This is\n subjective, but try to look at your changes with a critical eye. Do they fix\n errors in the original like misspellings or typos? Do they make existing\n prose more clear or accessible while maintaining accuracy? If you answered\n \"yes\" to either of those questions, this might be a great addition to our\n docs! If not, consider starting a discussion instead to see if your changes\n might be the exception to this guideline before submitting.\n- **Keep commits and pull requests small.** We get it. It's more convenient to\n throw all your changes into a single pull request or even into a single\n commit. The problem is that, if some of the changes are good and others don't\n quite work, having everything in one bucket makes it harder to filter out the\n great changes from those that need more work.\n- **Make spelling and grammar fixes in a separate pull request from any content\n changes.** These changes are quick to check and important to anyone reading\n the docs. We want to make sure they hit the live documentation as quickly as\n possible without being bogged down by other changes that require more\n intensive review.\n\nPlease see Gel's guide for `building documentation\n`_ from\nsource.\n\nDocumentation style\n~~~~~~~~~~~~~~~~~~~\n\n- **Lines should be no longer than 79 characters.**\n- **Remove trailing whitespace or whitespace on empty lines.**\n- **Surround references to parameter named with asterisks.** You may be tempted\n to surround parameter names with double backticks (````param````). We avoid\n that in favor of ``*param*``, in order to distinguish between parameter\n references and inline code (which *should* be surrounded by double\n backticks).\n- **Gel is singular.** Choose \"Gel is\" over \"Gel are\" and \"Gel\n does\" over \"Gel do.\"\n- **Use American English spellings.** Choose \"color\" over \"colour\" and\n \"organize\" over \"organise.\"\n- **Use the Oxford comma.** When delineating a series, place a comma between\n each item in the series, even the one with the conjunction. Use \"eggs, bacon,\n and juice\" rather than \"eggs, bacon and juice.\"\n- **Write in the simplest prose that is still accurate and expresses everything\n you need to convey.** You may be tempted to write documentation that sounds\n like a computer science textbook. Sometimes that's necessary, but in most\n cases, it isn't. Prioritize accuracy first and accessibility a close second.\n- **Be careful using words that have a special meaning in the context of\n Gel.** In casual speech or writing, you might talk about a \"set\" of\n something in a generic sense. Using the word this way in Gel documentation\n might easily be interpreted as a reference to Gel's `sets `.\n Avoid this kind of casual usage of key terms.\n" }, { "path": "Cargo.toml", "content": "[workspace]\nmembers = [\n \"edb/edgeql-parser\",\n \"edb/edgeql-parser/edgeql-parser-derive\",\n \"edb/edgeql-parser/edgeql-parser-python\",\n \"edb/graphql-rewrite\",\n \"edb/server/_rust_native\",\n \"rust/conn_pool\",\n \"rust/gel-http\",\n \"rust/pgrust\",\n \"rust/pyo3_util\",\n]\nresolver = \"2\"\n\n[workspace.dependencies]\npyo3 = { version = \"0.26\", features = [\"extension-module\", \"serde\", \"macros\"] }\ntokio = { version = \"1\", features = [\"rt\", \"rt-multi-thread\", \"macros\", \"time\", \"sync\", \"net\", \"io-util\"] }\ntracing = \"0.1.40\"\ntracing-subscriber = { version = \"0.3.20\", features = [\"registry\", \"env-filter\"] }\n\ngel-auth = { version = \"=0.1.6\" }\ngel-stream = { version = \"=0.4.3\" }\ngel-protocol = { version = \"=0.8.5\" }\ngel-jwt = { version = \"=0.1.4\" }\ngel-db-protocol = { version = \"=0.1.2\" }\ngel-pg-protocol = { version = \"=0.1.1\" }\ngel-pg-captive = { version = \"=0.1.1\" }\ngel-dsn = { version = \"=0.2.14\" }\n\nconn_pool = { path = \"rust/conn_pool\" }\npgrust = { path = \"rust/pgrust\" }\ngel-http = { path = \"rust/gel-http\" }\npyo3_util = { path = \"rust/pyo3_util\" }\n\n[profile.release]\ndebug = true\nlto = true\n\n[workspace.lints.rust]\nunexpected_cfgs = { level = \"warn\", check-cfg = ['cfg(never)'] }\n\n[patch.crates-io]\nopenssl-probe = { git = \"https://github.com/edgedb/openssl-probe/\", rev = \"e5ed593600d1f8128629565d349682f54b3a8b57\" }\n" }, { "path": "LICENSE", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "MANIFEST.in", "content": "recursive-include edb *.edgeql *.esdl *.py *.txt\nrecursive-include tests *.edgeql *.esdl *.py\ninclude LICENSE README.md logo.svg\n\nrecursive-include edb/edgeql-parser *\nrecursive-include edb/edgeql-parser/edgeql-parser-python *\nrecursive-include edb/server/protocol/auth_ext/_static *\n" }, { "path": "Makefile", "content": ".PHONY: build docs cython postgres postgres-ext pygments build-reqs\n.DEFAULT_GOAL := build\n\nSPHINXOPTS:=\"-W -n\"\n\nBUILD_REQS_SCRIPT='print(\"\\x00\".join(__import__(\"build\").ProjectBuilder(\".\").build_system_requires))'\n\nbuild-reqs:\n\tpython -m pip install --no-build-isolation build\n\tpython -c $(BUILD_REQS_SCRIPT) | xargs -0 python -m pip install --no-build-isolation\n\n\ncython: build-reqs\n\tfind edb -name '*.pyx' | xargs touch\n\tBUILD_EXT_MODE=py-only python setup.py build_ext --inplace\n\n\n# Just rebuild actually changed cython. This *should* work, since\n# that is how build systems are supposed to be, but it sometimes\n# fails in annoying ways.\ncython-fast: build-reqs\n\tBUILD_EXT_MODE=py-only python setup.py build_ext --inplace\n\n\nrust: build-reqs\n\tBUILD_EXT_MODE=rust-only python setup.py build_ext --inplace\n\n\ncli: build-reqs\n\tpython setup.py build_cli\n\n\ndocs: build-reqs\n\tfind docs -name '*.rst' | xargs touch\n\t$(MAKE) -C docs html SPHINXOPTS=$(SPHINXOPTS) BUILDDIR=\"../build\"\n\n\npostgres: build-reqs\n\tpython setup.py build_postgres\n\n\nparsers:\n\tpython setup.py build_parsers --inplace\n\n\nlibpg-query:\n\tpython setup.py build_libpg_query\n\n\nui: build-reqs\n\tpython setup.py build_ui\n\n\npygments: build-reqs\n\tout=$$(edb gen-meta-grammars edgeql) && \\\n\t\techo \"$$out\" > edb/tools/pygments/edgeql/meta.py\n\n\ncasts: build-reqs\n\tout=$$(edb gen-cast-table) && \\\n\t\techo \"$$out\" > docs/reference/edgeql/casts.csv\n\n\nbuild: build-reqs\n\tfind edb -name '*.pyx' | xargs touch\n\tpip install --upgrade --editable .[docs,test,language-server]\n\n\nclean:\n\tgit clean -Xfd -e \"!/*.code-workspace\" -e \"!/*.vscode\"\n" }, { "path": "NOTICE", "content": "EdgeDB\nCopyright 2008-present EdgeDB Inc.\n\nThis product includes software developed by\nEdgeDB Inc (https://www.edgedb.com/).\n" }, { "path": "README.md", "content": "\n\n
\n

Gel

\n \n \"Stars\"\n \n \n \n \n \n \"license\"\n \n \n \"discord\"\n \n
\n
\n Learn: build an app with Gel\n   ā€¢  \n Website\n   ā€¢  \n Docs\n   ā€¢  \n Blog\n   ā€¢  \n Discord\n   ā€¢  \n Twitter\n
\n\n
\n\n
\n
\n\n
\n
\n

What is Gel?

\n

\n Gel is a new kind of database\n
\n that takes the best parts of\n
\n relational databases, graph\n
\n databases, and ORMs. We call it\n
a graph-relational database.\n

\n
\n\n
\n\n
\n
\n

šŸ§© Types, not tables šŸ§©

\n
\n
\n\nSchema is the foundation of your application. It should be something you can\nread, write, and understand.\n\nForget foreign keys; tabular data modeling is a relic of an older age, and it\n[isn't compatible](https://en.wikipedia.org/wiki/Object%E2%80%93relational_impedance_mismatch)\nwith modern languages. Instead, Gel thinks about schema the same way you do:\nas **object types** containing **properties** connected by **links**.\n\n```esdl\ntype Person {\n required name: str;\n}\n\ntype Movie {\n required title: str;\n multi actors: Person;\n}\n```\n\nThis example is intentionally simple, but Gel supports everything you'd\nexpect from your database: a strict type system, indexes, constraints, computed\nproperties, stored procedures...the list goes on. Plus it gives you some shiny\nnew features too: link properties, schema mixins, and best-in-class JSON\nsupport. Read the [schema docs](https://docs.geldata.com/reference/datamodel)\nfor details.\n\n\n\n
\n
\n

šŸŒ³ Objects, not rows šŸŒ³

\n
\n
\n\nGel's super-powered query language EdgeQL is designed as a ground-up\nredesign of SQL. EdgeQL queries produce rich, structured objects, not flat\nlists of rows. Deeply fetching related objects is painless...bye, bye, JOINs.\n\n```esdl\nselect Movie {\n title,\n actors: {\n name\n }\n}\nfilter .title = \"The Matrix\"\n```\n\nEdgeQL queries are also _composable_; you can use one EdgeQL query as an\nexpression inside another. This property makes things like _subqueries_ and\n_nested mutations_ a breeze.\n\n```esdl\ninsert Movie {\n title := \"The Matrix Resurrections\",\n actors := (\n select Person\n filter .name in {\n 'Keanu Reeves',\n 'Carrie-Anne Moss',\n 'Laurence Fishburne'\n }\n )\n}\n```\n\nThere's a lot more to EdgeQL: a comprehensive standard library, computed\nproperties, polymorphic queries, `with` blocks, transactions, and much more.\nRead the [EdgeQL docs](https://docs.geldata.com/reference/edgeql) for the full\npicture.\n\n
\n
\n

šŸ¦‹ More than a mapper šŸ¦‹

\n
\n
\n\nWhile Gel solves the same problems as ORM libraries, it's so much more. It's\na full-fledged database with a\n[powerful and elegant query language](https://docs.geldata.com/reference/edgeql), a\n[migrations system](https://docs.geldata.com/learn/migrations), a\n[suite of client libraries](https://docs.geldata.com/reference/clients) in\ndifferent languages, a\n[command line tool](https://docs.geldata.com/learn/cli), and a managed\n[cloud service](https://geldata.com/cloud).\nThe goal is to rethink every aspect of how developers model, migrate,\nmanage, and query their database.\n\nHere's a taste-test of Gel's next-level developer experience: you can\ninstall our CLI, spin up an instance, and open an interactive EdgeQL shell with\njust three commands.\n\n```\n$ curl --proto '=https' --tlsv1.2 -sSf https://geldata.com/sh | sh\n$ edgedb project init\n$ edgedb\nedgedb> select \"Hello world!\"\n```\n\nWindows users: use this Powershell command to install the CLI.\n\n```\nPS> iwr https://geldata.com/ps1 -useb | iex\n```\n\n
\n\n## Get started\n\nTo start learning about Gel, check out the following resources:\n\n- **[The quickstart](https://docs.geldata.com/learn/quickstart/overview/nextjs)**. If\n you're just starting out, the 10-minute quickstart guide is the fastest way\n to get up and running.\n- **[Gel Cloud šŸŒ¤ļø](https://www.geldata.com/cloud)**. The best\n most effortless way to host your Gel database in the cloud.\n\n- **The docs.** Jump straight into the docs for\n [schema modeling](https://docs.geldata.com/reference/datamodel) or\n [EdgeQL](https://docs.geldata.com/reference/edgeql)!\n\n
\n\n## Contributing\n\nPRs are always welcome! To get started, follow\n[this guide](https://docs.geldata.com/resources/guides/contributing) to build Gel from\nsource on your local machine.\n\n[File an issue šŸ‘‰](https://github.com/geldata/gel/issues/new/choose)\n
\n[Start a Discussion šŸ‘‰](https://github.com/geldata/gel/discussions/new)\n
\n[Join the discord šŸ‘‰](https://discord.gg/gel)\n\n
\n\n## License\n\nThe code in this repository is developed and distributed under the\nApache 2.0 license. See [LICENSE](LICENSE) for details.\n" }, { "path": "build_backend.py", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2008-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n# This is a straight proxy to setuptools.build_meta backend that exists\n# solely because someone thought that in-tree build dependencies should\n# require this.\n\nfrom setuptools.build_meta import * # noqa\n" }, { "path": "dev-notes/concurrent-indexes.py", "content": "#!/usr/bin/env python3\n\nimport gel\n\n\ndef create_concurrent_indexes(db, msg_callback=print):\n '''Actually create all \"create concurrently\" indexes\n\n The protocol here is to find all the indexes that need created,\n and create them with `administer concurrent_index_build()`.\n It's possible that the database will shut down after an index\n creation but before the metadata is updated, in which case\n we might rerun the command later, which is harmless.\n\n If we stick with this ADMINISTER-based schemed, I figure this code\n would live in the CLI.\n '''\n indexes = db.query('''\n select schema::Index {\n id, expr, subject_name := .{index.id}\")\n ''')\n\n\ndef main():\n with gel.create_client() as db:\n create_concurrent_indexes(db)\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "dev-notes/inplace-upgrades.md", "content": "The inplace upgrade system adds three new flags to edgedb-server. They may (though probably usually won't) be specified together. If any of them is specified, the server will exit after performing the in-place upgrade operations instead of starting up.\n\n * ``-inplace-upgrade-prepare `` -- \"prepare\" an inplace upgrade, using schema information provided in ````. (More about this later.) This will create the new standard library (in a namespace), populate the schema tables with user schemas, and prepare (but not execute) any irreversible scripts for updating the standard library trampolines and fixing up user-defined functions.\n\n This operation should not do anything that cannot be backed out.\n It may be run while an older version of the server is still live.\n\n If this is interrupted, crashes, or fails, it *will leave a partially prepared database*. To deal with this, see the next command.\n\n The file should be in the format produced by ``tests/inplace-testing/prep-upgrades.py``: a JSON object where the keys are branch names and the values are the results of executing ``administer prepare_upgrade()``.\n\n * ``-inplace-upgrade-rollback`` -- Rolls back a prepared upgrade.\n This works by deleting everything in the newly created schemas. It can rollback partially prepared upgrades.\n\n It may be run while an older version of the server is still live.\n\n * ``-inplace-upgrade-finalize`` -- Finalizes a prepared upgrade by fully flipping the database to the new version. This flips standard library trampolines, patches user-defined functions, and deletes the old standard library.\n\n The old version must not be running. (Though there is not a clear way to enforce this.)\n\n Finalize does a dry run of each branch's upgrade inside a transaction before making any changes. If this fails, the upgrade may be broken (due to a bug or an incompatibility), and it may still be rolled back.\n\n If finalize fails *after* the dry run, once it has started actually finalizing branches, then it *may not* be rolled back. Because all of the upgrades were tested in (reverted) transactions, this *should* only happen in the case of interruption or postgres crash, and it should be safe to retry the finalize.\n\n If finalize emitted a message of the form \"Finished pivoting branch ''\", then the upgrade may not be rolled back; the only way out is through. Rollback will refuse to operate in this case.\n\n-----\n\nSuggested procedure:\n\n0.5. Make a backup\n\n1. ``edgedb query 'configure instance set force_database_error := $${\"type\": \"AvailabilityError\", \"message\": \"DDL is disabled due to in-place upgrade.\", \"_scopes\": [\"ddl\"]}$$;'``.\n This will disable all DDL commands to the database, while leaving it running for both read and write queries.\n\n2. ``tests/inplace-testing/prep-upgrades.py > \"upgrade.json\"``.\n This will dump the information needed for upgrade.\n\n3. ``edgedb-server --backend-dsn=\"$DSN\" --inplace-upgrade-prepare upgrade.json``.\n This will prepare the upgrade.\n\n4. Stop the old edgedb server.\n\n4.5. Make a backup\n\n5. ``edgedb-server --backend-dsn=\"$DSN\" --inplace-upgrade-finalize``.\n This will finalize the upgrade.\n\n6. Start the new server.\n\n7. ``edgedb query 'configure instance reset force_database_error'``\n\nIf there is a failure in step 3 or step 5 *before* a branch has finished pivoting, then it can be rolled back with ``edgedb-server --backend-dsn=\"$DSN\" --inplace-upgrade-rollback``.\n\nIf there is a failure after a branch has been pivoted, then there is nothing to do but retry it.\n(And restore from a backup if that doesn't work. That would be a bug, and one that has slipped past at least one line of defence.)\n\n\n----\n\nTesting notes:\n\nCurrently, we can only inplace upgrade beween full major versions, since we use the major version number to distinguish between the namespaced stdlibs.\n\nFor testing inplace upgrades, we have a test that applies a patch that bumps the major version number and catalog.\n\nTODO: Maybe we should use the catalog number instead, which will make it easier to test between different nightlies.\n" }, { "path": "dev-notes/newtype-checklist.md", "content": "This is a checklist of steps needed to add a new type to EdgeDB, along with links to examples of PRs doing the tasks.\n\nCore database range PRs:\n * https://github.com/edgedb/edgedb/pull/3983\n * https://github.com/edgedb/edgedb/pull/4020\n\nCore database cal::duration PRs:\n * https://github.com/edgedb/edgedb/pull/3948\n\n- [ ] JSON handling\n - [ ] Implement JSON casts if the default Postgres behavior won't work\n - [ ] Update output.serialize_expr_to_json(), if the default won't work\n * range: https://github.com/edgedb/edgedb/pull/4008\n\n- [ ] If any new functions or constructors have an implementation that is not\n just purely a call to a strict function, make sure to test with inputs\n that are NULL at runtime!\n Probably the easiest way to generate NULL-at-runtime values is\n `$0` and then passing in `{}`.\n * range test example and bugfix: https://github.com/edgedb/edgedb/pull/4207/\n\n- [ ] For compound types, add a schema class in edb/schema/types.py and\n\n- [ ] Add mapping to pgsql types in edb/pgsql/types.py\n\n- [ ] Add implementations of any relevant functions/operations to `edb/lib`.\n\n- [ ] For compound types, add a type descriptor and code for encoding it in\nedb/server/compiler/sertypes.py.\n * range: https://github.com/edgedb/edgedb/pull/4016\n\n- [ ] For new scalar types, add it to edb/api/types.txt and edb/graphql/types.py. Run `edb gen-types`.\n\n- [ ] Update all of the first-party language drivers (or get their owners to)\n - [ ] Python (Fantix/Elvis/Sully)\n * cal::date_duration: https://github.com/edgedb/edgedb-python/pull/335\n * range: https://github.com/edgedb/edgedb-python/pull/332/\n - [ ] Go (Frederick)\n * cal::date_duration: https://github.com/edgedb/edgedb-go/pull/232\n - [ ] Javascript (James/Colin)\n * cal::date_duration: https://github.com/edgedb/edgedb-js/pull/373/\n * range: https://github.com/edgedb/edgedb-js/pull/377\n - [ ] Rust/CLI (Paul)\n * This requires updating both the Rust bindings to support the\n new type and the CLI to properly print it\n * cal::date_duration: https://github.com/edgedb/edgedb-rust/pull/146, https://github.com/edgedb/edgedb-cli/pull/759\n * range: https://github.com/edgedb/edgedb-rust/pull/145, https://github.com/edgedb/edgedb-cli/pull/755\n\n- [ ] Add a field of the new type to the `dump` test for the new version\n\n- [ ] Write tests.\n" }, { "path": "dev-notes/release-process.md", "content": "# Instructions for releasing a new version\n\nDeprecates release instruction from\n[RFC 2](https://github.com/edgedb/rfcs/blob/master/text/0002-edgedb-release-process.rst).\n\nEdgeDB packages are published on https://packages.edgedb.com.\nThey are build in GitHub Actions pipelines, using\nhttps://github.com/edgedb/edgedb-pkg.\n\nReleases are built from a release branch associated with a major version\n(i.e. \"release/4.x\"). At feature freeze, we create this branch. From that moment\non, all additional commits will have to be cherry-picked to this branch.\n\nBefore the major version, we publish \"testing releases\":\n\n- \"alpha\" (i.e. `v4.0a1`, `v4.0a2`),\n- \"beta\" (i.e. `v4.0b1`, `v4.0b2`),\n- \"release candidates\" (`v4.0rc1`) that we might promote into the final release.\n\n## Internal Communication\n\nAnnounce on team slack when you are beginning to prepare a release,\nwhen a release build has been kicked off, and when the release has\nsucceeded. Update the thread with any problems and attempted\nresolutions.\n\nCommunicate in the other direction as well: make sure the release\nmanager knows of any pending work that you want in a release.\n\n\"b1\", \"rc1\", and \".0\" releases are big deals. Make sure to get signoff\nbefore releasing.\n\n## edgedb-ui\n\nOn release branches, `edgedb-ui` should be pinned to the associated branch.\nThis can be done in `setup.py` with the variable `EDGEDBGUI_COMMIT`.\nFor example, on branch `release/4.x`, it is pinned to `edgedb-ui`'s branch `4.x`.\nThis means any release off `release/4.x` will contain latest commits from\n`edgedb-ui`'s branch `4.x`.\n\n\n## Preparing commits for a release\n\nFor each major release `N`, we have two GitHub labels: `to-backport-N.x` and\n`backported-N.x`. PRs that need to be backported should be labelled with\n`to-backport-N.x` for each of the target versions.\n\nOnce a PR is backported, `to-backport-N.x` should be removed and\n`backported-N.x` added.\n\nTracking both states makes it easy to tell what needs to be backported\nand what has been backported.\n\n(Historical note: previously we had simply a `backport-N.x` label.\nThis made it easy to ensure that everything that got labelled with\n`backport` actually got backported, but there was not an at-a-glance\nway to see if something *had* been backported. Even looking at the\nissue didn't always tell you, since sometimes we labelled things as\n`backport` and then thought better of it.)\n\n### Technical helpers\n\nThe `gh` command line makes a bunch of these operations simple.\n\nTo enumerate all pending backports for a branch:\n```bash\ngh pr list --state all -l to-backport-5.x\n```\n\nTo adjust labels to mark a PR as backported:\n```bash\ngh pr edit --remove-label to-backport-N.x --add-label backported-N.x \n```\n\nA helper shell script to cherry-pick a commit using its PR number:\n\n```bash\n# this won't work if a PR is not squashed into a single commit\nfunction cp-pr {\n git cherry-pick $(gh pr view $1 --json mergeCommit --jq .mergeCommit.oid)\n}\n```\n\n\n### What to backport?\nSometimes, people will forget to label the PR to be back-ported, so a good\npractice is to list all commits since the last release:\n\n```\ngit show releases/4.x # to see the last commit that has been cherry-picked\ngit log master # find the hash of that commit on master\ngit log hash_of_that_commit..master > ../to-backport.txt\n```\n\nNow, one can go through the list and see if the commits are worth back-porting.\nA few pointers:\n\n- Don't backport new features, unless it is high-priority for some reason.\n- Don't backport docs, since the website is built from master.\n- Don't backport refactors, since they might introduce bugs and there is no\n point in improving the codebase of a branch we are not developing on anymore.\n Disregard this rule early on after the fork of the release branch, since\n porting refactors will decrease chances merge conflicts of other commits later\n on.\n- Don't backport \"build\" commits (updating of build deps, refactoring of the\n release pipeline), since that might trigger problems in the release process.\n- If a PR changes:\n\n - any of the schema objects (i.e. adding a field to `s_types.Type`) or\n - a std library object (i.e. changing implementation of `std::round`),\n - metaschema (i.e. changing a pg function `edgedb.range_to_jsonb`),\n ... a \"patch\" needs to be added into `pgsql/patches.py`.\n This is needed, because minor releases don't require a \"dump and restore\",\n so we must apply these changes to existing user databases.\n\n Patches must be tested using this GHA workflow:\n https://github.com/edgedb/edgedb/actions/workflows/tests-patches.yml\n\n## Release pipeline\n\nWhen you have your commits ready, tag the commit and push:\n\n```\n# git tag --sign v4.5\n# git push origin releases/4.x --follow-tags\n```\n\nThen open GitHub Actions page and run one of these pipelines:\n\n- https://github.com/edgedb/edgedb/actions/workflows/testing.yml\n- https://github.com/edgedb/edgedb/actions/workflows/release.yml\n\nThis will kick-off an GHA workflow that should take ~3 hours.\nIt will build, test and publish for each of the supported platforms.\nIt will not publish any packages if any of the tests fail.\n\nSometimes, tests will be flakey and just need to be re-run.\nYou can do that with a button top-right.\n\n## Changelog\n\nEach major release has a changelog page in the docs (i.e.\n`docs/changelog/4_x.rst`). It should contain explanations of the new features,\nwhich are usually composed by our dev-rel team.\n\nEach minor release is just a subsection in the page, as a list of back-ported\nPRs. Any PRs that fix internal stuff (like our test framework) or are not user\nfacing should not be included in the changelog.\n\nDon't forget to include commits released from `edgedb-ui`.\n\nThese changes need to land on master branch and are not needed on the release\nbranch, so best course of action if to open a PR to master after kicking off\nthe release pipeline. After that PR is merged, the website needs to be\ndeployed, for changelog to land on the website (ping dev rel team).\n\nA helper function to generate changelog is:\n\n```python\n# I keep this in ../compose-changelog.py\n\nimport json\nimport requests\nimport re\nimport sys\n\nBASE_URL = 'https://api.github.com/repos/edgedb/edgedb/compare'\n\ndef main():\n if len(sys.argv) < 2:\n print('pass a sha1 hash as a first argument')\n sys.exit(1)\n\n from_hash = sys.argv[1]\n if len(sys.argv) > 2:\n to_hash = sys.argv[2]\n\n r = requests.get(f'{BASE_URL}/{from_hash}...{to_hash}')\n data = json.loads(r.text)\n\n for commit in data['commits']:\n message = commit['commit']['message']\n first_line = message.partition('\\n\\n')[0]\n if commit.get('author'):\n username = '@{}'.format(commit['author']['login'])\n else:\n username = commit['commit']['author']['name']\n sha = commit[\"sha\"][:8]\n\n m = re.search(r'\\#(?P\\d+)\\b', message)\n if m:\n issue_num = m.group('num')\n else:\n issue_num = None\n\n first_line = re.sub(r'\\(\\#(?P\\d+)\\)', '', first_line)\n print(f'* {first_line}')\n # print(f' (by {username} in {sha}', end='')\n if issue_num:\n print(f' (:eql:gh:`#{issue_num}`)')\n print()\n\nif __name__ == '__main__':\n main()\n```\n\n```bash\npython ../compose-changelog.py v4.5 v4.6 >> docs/changelog/4_x.rst\n```\n\n## After the release\n\nThe release pipelines will make the new version available at\nhttps://packages.edgedb.com. This is enough for it to be installable using the\nCLI, but other methods of installation need to be kicked of manually:\n\n- our cloud team needs to deploy separate _cloud wizardly groups_,\n- docker image needs to be published to https://hub.docker.com,\n- Digital Ocean image needs to be published by Frederick,\n" }, { "path": "docs/.gitignore", "content": "/_build\n" }, { "path": "docs/Makefile", "content": "# Makefile for Sphinx documentation\n#\n\n# You can set these variables from the command line.\nSPHINXOPTS =\nSPHINXBUILD = sphinx-build\nPAPER =\nBUILDDIR = _build\n\n# User-friendly check for sphinx-build\nifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)\n$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)\nendif\n\n# Internal variables.\nPAPEROPT_a4 = -D latex_paper_size=a4\nPAPEROPT_letter = -D latex_paper_size=letter\nALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .\n# the i18n builder cannot share the environment and doctrees with the others\nI18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .\n\n.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext\n\nhelp:\n\t@echo \"Please use \\`make ' where is one of\"\n\t@echo \" html to make standalone HTML files\"\n\t@echo \" dirhtml to make HTML files named index.html in directories\"\n\t@echo \" singlehtml to make a single large HTML file\"\n\t@echo \" pickle to make pickle files\"\n\t@echo \" json to make JSON files\"\n\t@echo \" htmlhelp to make HTML files and a HTML help project\"\n\t@echo \" qthelp to make HTML files and a qthelp project\"\n\t@echo \" applehelp to make an Apple Help Book\"\n\t@echo \" devhelp to make HTML files and a Devhelp project\"\n\t@echo \" epub to make an epub\"\n\t@echo \" latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter\"\n\t@echo \" latexpdf to make LaTeX files and run them through pdflatex\"\n\t@echo \" latexpdfja to make LaTeX files and run them through platex/dvipdfmx\"\n\t@echo \" text to make text files\"\n\t@echo \" man to make manual pages\"\n\t@echo \" texinfo to make Texinfo files\"\n\t@echo \" info to make Texinfo files and run them through makeinfo\"\n\t@echo \" gettext to make PO message catalogs\"\n\t@echo \" changes to make an overview of all changed/added/deprecated items\"\n\t@echo \" xml to make Docutils-native XML files\"\n\t@echo \" pseudoxml to make pseudoxml-XML files for display purposes\"\n\t@echo \" linkcheck to check all external links for integrity\"\n\t@echo \" doctest to run all doctests embedded in the documentation (if enabled)\"\n\t@echo \" coverage to run coverage check of the documentation (if enabled)\"\n\nclean:\n\trm -rf $(BUILDDIR)/*\n\nhtml:\n\t$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html\n\t@echo\n\t@echo \"Build finished. The HTML pages are in $(BUILDDIR)/html.\"\n\ndirhtml:\n\t$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml\n\t@echo\n\t@echo \"Build finished. The HTML pages are in $(BUILDDIR)/dirhtml.\"\n\nsinglehtml:\n\t$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml\n\t@echo\n\t@echo \"Build finished. The HTML page is in $(BUILDDIR)/singlehtml.\"\n\npickle:\n\t$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle\n\t@echo\n\t@echo \"Build finished; now you can process the pickle files.\"\n\njson:\n\t$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json\n\t@echo\n\t@echo \"Build finished; now you can process the JSON files.\"\n\nhtmlhelp:\n\t$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp\n\t@echo\n\t@echo \"Build finished; now you can run HTML Help Workshop with the\" \\\n\t \".hhp project file in $(BUILDDIR)/htmlhelp.\"\n\nqthelp:\n\t$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp\n\t@echo\n\t@echo \"Build finished; now you can run \"qcollectiongenerator\" with the\" \\\n\t \".qhcp project file in $(BUILDDIR)/qthelp, like this:\"\n\t@echo \"# qcollectiongenerator $(BUILDDIR)/qthelp/EdgeDB.qhcp\"\n\t@echo \"To view the help file:\"\n\t@echo \"# assistant -collectionFile $(BUILDDIR)/qthelp/EdgeDB.qhc\"\n\napplehelp:\n\t$(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp\n\t@echo\n\t@echo \"Build finished. The help book is in $(BUILDDIR)/applehelp.\"\n\t@echo \"N.B. You won't be able to view it unless you put it in\" \\\n\t \"~/Library/Documentation/Help or install it in your application\" \\\n\t \"bundle.\"\n\ndevhelp:\n\t$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp\n\t@echo\n\t@echo \"Build finished.\"\n\t@echo \"To view the help file:\"\n\t@echo \"# mkdir -p $$HOME/.local/share/devhelp/EdgeDB\"\n\t@echo \"# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/EdgeDB\"\n\t@echo \"# devhelp\"\n\nepub:\n\t$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub\n\t@echo\n\t@echo \"Build finished. The epub file is in $(BUILDDIR)/epub.\"\n\nlatex:\n\t$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex\n\t@echo\n\t@echo \"Build finished; the LaTeX files are in $(BUILDDIR)/latex.\"\n\t@echo \"Run \\`make' in that directory to run these through (pdf)latex\" \\\n\t \"(use \\`make latexpdf' here to do that automatically).\"\n\nlatexpdf:\n\t$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex\n\t@echo \"Running LaTeX files through pdflatex...\"\n\t$(MAKE) -C $(BUILDDIR)/latex all-pdf\n\t@echo \"pdflatex finished; the PDF files are in $(BUILDDIR)/latex.\"\n\nlatexpdfja:\n\t$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex\n\t@echo \"Running LaTeX files through platex and dvipdfmx...\"\n\t$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja\n\t@echo \"pdflatex finished; the PDF files are in $(BUILDDIR)/latex.\"\n\ntext:\n\t$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text\n\t@echo\n\t@echo \"Build finished. The text files are in $(BUILDDIR)/text.\"\n\nman:\n\t$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man\n\t@echo\n\t@echo \"Build finished. The manual pages are in $(BUILDDIR)/man.\"\n\ntexinfo:\n\t$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo\n\t@echo\n\t@echo \"Build finished. The Texinfo files are in $(BUILDDIR)/texinfo.\"\n\t@echo \"Run \\`make' in that directory to run these through makeinfo\" \\\n\t \"(use \\`make info' here to do that automatically).\"\n\ninfo:\n\t$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo\n\t@echo \"Running Texinfo files through makeinfo...\"\n\tmake -C $(BUILDDIR)/texinfo info\n\t@echo \"makeinfo finished; the Info files are in $(BUILDDIR)/texinfo.\"\n\ngettext:\n\t$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale\n\t@echo\n\t@echo \"Build finished. The message catalogs are in $(BUILDDIR)/locale.\"\n\nchanges:\n\t$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes\n\t@echo\n\t@echo \"The overview file is in $(BUILDDIR)/changes.\"\n\nlinkcheck:\n\t$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck\n\t@echo\n\t@echo \"Link check complete; look for any errors in the above output \" \\\n\t \"or in $(BUILDDIR)/linkcheck/output.txt.\"\n\ndoctest:\n\t$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest\n\t@echo \"Testing of doctests in the sources finished, look at the \" \\\n\t \"results in $(BUILDDIR)/doctest/output.txt.\"\n\ncoverage:\n\t$(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage\n\t@echo \"Testing of coverage in the sources finished, look at the \" \\\n\t \"results in $(BUILDDIR)/coverage/python.txt.\"\n\nxml:\n\t$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml\n\t@echo\n\t@echo \"Build finished. The XML files are in $(BUILDDIR)/xml.\"\n\npseudoxml:\n\t$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml\n\t@echo\n\t@echo \"Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml.\"\n" }, { "path": "docs/cloud/cli.rst", "content": ".. _ref_guide_cloud_cli:\n\n===\nCLI\n===\n\n:edb-alt-title: Using Gel Cloud via the CLI\n\nTo use |Gel| Cloud via the CLI, first log in using\n:ref:`ref_cli_gel_cloud_login`.\n\n.. note::\n\n This is the way you'll log in interactively on your development machine,\n but when interacting with Gel Cloud via a script or in CI, you'll\n instead set the :gelenv:`SECRET_KEY` environment variable to your secret\n key. Generate a secret key in the Gel Cloud UI or by running\n :ref:`ref_cli_gel_cloud_secretkey_create`. The :gelcmd:`cloud\n login` and :gelcmd:`cloud logout` commands are not intended for use\n in this context.\n\nOnce your login is successful, you will be able to create an instance using\neither :ref:`ref_cli_gel_instance_create` or\n:ref:`ref_cli_gel_project_init`, depending on whether you also want to\ncreate a local project linked to your instance.\n\n* :ref:`ref_cli_gel_instance_create` with an instance name of\n ``/``.\n\n .. code-block:: bash\n\n $ gel instance create /\n\n* :ref:`ref_cli_gel_project_init` with the ``--server-instance`` option. Set\n the server instance name to ``/``.\n\n .. code-block:: bash\n\n $ gel project init \\\n --server-instance /\n\n Alternatively, you can run :gelcmd:`project init` *without* the\n ``--server-instance`` option and enter an instance name in the\n ``/`` format when prompted interactively.\n\n.. note::\n\n Please be aware of the following restrictions on |Gel| Cloud instance\n names:\n\n * can contain only Latin alpha-numeric characters or ``-``\n * cannot start with a dash (``-``) or contain double dashes (``--``)\n * maximum instance name length is 61 characters minus the length of your\n organization name (i.e., length of organization name + length of instance\n name must be fewer than 62 characters)\n\nTo use :gelcmd:`instance create`:\n\n.. code-block:: bash\n\n $ gel instance create /\n\nTo use :gelcmd:`project init`:\n\n.. code-block:: bash\n\n $ gel project init \\\n --server-instance /\n\nAlternatively, you can run :gelcmd:`project init` *without* the\n``--server-instance`` option and enter an instance name in the\n``/`` format when prompted interactively.\n" }, { "path": "docs/cloud/deploy/fly.rst", "content": ".. _ref_guide_cloud_deploy_fly:\n\n======\nFly.io\n======\n\n:edb-alt-title: Deploying applications built on Gel Cloud to Fly.io\n\n1. Install the `Fly.io CLI `_\n2. Log in to Fly.io with ``flyctl auth login``\n3. Run ``flyctl launch`` to create a new app on Fly.io and configure it.\n It will ask you to select a region and a name for your app. When done it will\n create a ``fly.toml`` file and a ``Dockerfile`` in your project directory.\n4. Set :gelenv:`INSTANCE` and :gelenv:`SECRET_KEY` as secrets in your Fly.io\n app.\n\n For **runtime secrets**, you can do this by running the following commands:\n\n .. code-block:: bash\n\n $ flyctl secrets set GEL_INSTANCE \n $ flyctl secrets set GEL_SECRET_KEY \n\n `Read more about Fly.io runtime secrets\n `_.\n\n For **build secrets**, you can do this by modifying the ``Dockerfile`` to\n mount the secrets as environment variables.\n\n .. code-block:: dockerfile-diff\n :caption: Dockerfile\n\n # Build application\n - RUN pnpm run build\n + RUN --mount=type=secret,id=GEL_INSTANCE \\\n + --mount=type=secret,id=GEL_SECRET_KEY \\\n + GEL_INSTANCE=\"$(cat /run/secrets/GEL_INSTANCE)\" \\\n + GEL_SECRET_KEY=\"$(cat /run/secrets/GEL_SECRET_KEY)\" \\\n + pnpm run build\n\n `Read more about Fly.io build secrets\n `_.\n\n5. Deploy your app to Fly.io\n\n .. code-block:: bash\n\n $ flyctl deploy\n\n If your app requires build secrets, you can pass them as arguments\n to the ``deploy`` command:\n\n .. code-block:: bash\n\n $ flyctl deploy --build-secret GEL_INSTANCE=\"\" \\\n --build-secret GEL_SECRET_KEY=\"\"\n" }, { "path": "docs/cloud/deploy/index.rst", "content": ".. _ref_guide_cloud_deploy:\n\n=============\nDeploy an app\n=============\n\n:edb-alt-title: Deploying applications built on Gel Cloud\n\nFor your production deployment, generate a dedicated secret key for your\ninstance with :ref:`ref_cli_gel_cloud_secretkey_create` or via the web UI's\n\"Secret Keys\" pane in your instance dashboard. Create two environment variables\naccessible to your production application:\n\n* :gelenv:`SECRET_KEY`- contains the secret key you generated\n* :gelenv:`INSTANCE`- the name of your Gel Cloud instance\n (``/``)\n\nIf you use one of these platforms, try the platform's guide for\nplatform-specific instructions:\n\n.. toctree::\n :maxdepth: 1\n\n vercel\n netlify\n fly\n railway\n render\n" }, { "path": "docs/cloud/deploy/netlify.rst", "content": ".. _ref_guide_cloud_deploy_netlify:\n\n=======\nNetlify\n=======\n\n:edb-alt-title: Deploying applications built on Gel Cloud to Netlify\n\n.. note::\n\n This guide assumes the Git deployment method on Netlify, but you may also\n deploy your site using other methods. Just make sure the Gel Cloud\n environment variables are set, and your app should have connectivity to\n your instance.\n\n1. Push project to GitHub or some other Git remote repository\n2. Create and make note of a secret key for your Gel Cloud instance\n3. On your Netlify Team Overview view under Sites, click Import from Git\n4. Import your project's repository\n5. Configure the build settings appropriately for your app\n6. Click the Add environment variable button\n7. Use the New variable button to add two variables:\n\n - :gelenv:`INSTANCE` containing your Gel Cloud instance name (in\n ``/`` format)\n - :gelenv:`SECRET_KEY` containing the secret key you created and noted\n previously.\n\n8. Click Deploy\n\n.. image:: images/cloud-netlify-config.png\n :width: 100%\n :alt: A screenshot of the Netlify deployment configuration view\n highlighting the environment variables section where a user will\n need to set the necessary variables for Gel Cloud instance\n connection.\n" }, { "path": "docs/cloud/deploy/railway.rst", "content": ".. _ref_guide_cloud_deploy_railway:\n\n=======\nRailway\n=======\n\n:edb-alt-title: Deploying applications built on Gel Cloud to Railway\n\n1. Push project to GitHub or some other Git remote repository\n2. Create and make note of a secret key for your Gel Cloud instance\n3. From Railway's dashboard, click the \"New Project\" button\n4. Select the repository you want to deploy\n5. Click the \"Add variables\" button to add the following environment variables:\n\n - :gelenv:`INSTANCE` containing your Gel Cloud instance name (in\n ``/`` format)\n - :gelenv:`SECRET_KEY` containing the secret key you created and noted\n previously.\n\n6. Click \"Deploy\"\n\n.. image:: images/cloud-railway-config.png\n :width: 100%\n :alt: A screenshot of the Railway deployment configuration view\n highlighting the environment variables section where a user will\n need to set the necessary variables for Gel Cloud instance\n connection.\n" }, { "path": "docs/cloud/deploy/render.rst", "content": ".. _ref_guide_cloud_deploy_render:\n\n======\nRender\n======\n\n:edb-alt-title: Deploying applications built on Gel Cloud to Render\n\n1. Push project to GitHub or some other Git remote repository\n2. Create and make note of a secret key for your Gel Cloud instance\n3. From Render's dashboard, click \"New > Web Service\"\n4. Import your project's repository\n5. In the setup page, scroll down to the \"Environment Variables\" section and\n add the following environment variables:\n\n - :gelenv:`INSTANCE` containing your Gel Cloud instance name (in\n ``/`` format)\n - :gelenv:`SECRET_KEY` containing the secret key you created and noted\n previously.\n\n6. Click Deploy\n\n.. image:: images/cloud-render-config.png\n :width: 100%\n :alt: A screenshot of the Render deployment configuration view\n highlighting the environment variables section where a user\n will need to set the necessary variables for Gel Cloud instance\n connection.\n" }, { "path": "docs/cloud/deploy/vercel.rst", "content": ".. _ref_guide_cloud_deploy_vercel:\n\n======\nVercel\n======\n\n:edb-alt-title: Deploying applications built on Gel Cloud to Vercel\n\n1. Push project to GitHub or some other Git remote repository\n2. Create and make note of a secret key for your Gel Cloud instance\n3. From Vercel's Overview tab, click Add New > Project\n4. Import your project's repository\n5. In \"Configure Project,\" expand \"Environment Variables\" to add two variables:\n\n - :gelenv:`INSTANCE` containing your Gel Cloud instance name (in\n ``/`` format)\n - :gelenv:`SECRET_KEY` containing the secret key you created and noted\n previously.\n\n6. Click Deploy\n\n.. image:: images/cloud-vercel-config.png\n :width: 100%\n :alt: A screenshot of the Vercel deployment configuration view highlighting\n the environment variables section where a user will need to set the\n necessary variables for |Gel| Cloud instance connection.\n" }, { "path": "docs/cloud/http_gql.rst", "content": ".. _ref_guide_cloud_http_gql:\n\n===================\nHTTP & GraphQL APIs\n===================\n\n:edb-alt-title: Querying Gel Cloud over HTTP and GraphQL\n\nUsing |Gel| Cloud via HTTP and GraphQL works the same as :ref:`using any other\n|Gel| instance `. The two differences are in **how to\ndiscover your instance's URL** and **authentication**.\n\n\nEnabling\n========\n\n|Gel| Cloud can expose an HTTP endpoint for EdgeQL queries. Since HTTP is a\nstateless protocol, no :ref:`DDL ` or :ref:`transaction commands\n`, can be executed using this endpoint. Only one\nquery per request can be executed.\n\nIn order to set up HTTP access to the database add the following to\nthe schema:\n\n.. code-block:: sdl\n\n using extension edgeql_http;\n\nThen create a new migration and apply it using\n:ref:`ref_cli_gel_migration_create` and\n:ref:`ref_cli_gel_migrate`, respectively.\n\nYour instance can now receive EdgeQL queries over HTTP at\n``https://:/branch//edgeql``.\n\n\nInstance URL\n============\n\nTo determine the URL of a |Gel| Cloud instance, find the host by running\n:gelcmd:`instance credentials -I /`. Use the\n``host`` and ``port`` from that table in the URL format above this note.\nChange the protocol to ``https`` since Gel Cloud instances are secured\nwith TLS.\n\nYour instance can now receive EdgeQL queries over HTTP at\n``https://:/branch//edgeql``.\n\n\nAuthentication\n==============\n\n\nTo authenticate to your |Gel| Cloud instance, first create a secret key using\nthe Gel Cloud UI or :ref:`ref_cli_gel_cloud_secretkey_create`. Use the\nsecret key as your token with the bearer authentication method. Here is an\nexample showing how you might send the query ``select Person {*};`` using cURL:\n\n.. lint-off\n\n.. code-block:: bash\n\n $ curl -G https://:/branch/main/edgeql \\\n -H \"Authorization: Bearer \\\n --data-urlencode \"query=select Person {*};\"\n\n.. lint-on\n\n\nUsage\n=====\n\nUsage of the HTTP and GraphQL APIs is identical on a |Gel| Cloud instance.\nReference the HTTP and GraphQL documentation for more information.\n\n\nHTTP\n----\n\n- :ref:`Overview `\n- :ref:`ref_edgeql_protocol`\n- :ref:`ref_edgeql_http_health_checks`\n\n\nGraphQL\n-------\n\n- :ref:`Overview `\n- :ref:`ref_graphql_overview`\n- :ref:`ref_graphql_mutations`\n- :ref:`ref_graphql_introspection`\n- :ref:`ref_cheatsheet_graphql`\n" }, { "path": "docs/cloud/index.rst", "content": ".. _ref_guide_cloud:\n\n=====\nCloud\n=====\n\n:edb-alt-title: Using Gel Cloud\n\n.. toctree::\n :maxdepth: 2\n :hidden:\n\n cli\n web\n http_gql\n deploy/index\n deploy/vercel\n deploy/netlify\n deploy/fly\n deploy/render\n deploy/railway\n migrate_from\n\n\n|Gel| Cloud is a fully managed, effortless cloud database service,\nengineered to let you deploy your database instantly and connect from\nanywhere with near-zero configuration.\n\nConnecting your app\n===================\n\nTry a guide for connecting your app running on your platform of choice:\n\n.. TODO: render these with icons\n\n* :ref:`Vercel `\n* :ref:`Netlify `\n* :ref:`Fly.io `\n* :ref:`Render `\n* :ref:`Railway `\n\nTo connect your apps running on other platforms, generate a dedicated\nsecret key for your instance with :gelcmd:`cloud secretkey create` or via the\nweb UI's ā€œSecret Keysā€ pane in your instance dashboard. Create two environment\nvariables accessible to your production application:\n\n* :gelenv:`SECRET_KEY` - contains the secret key you generated\n* :gelenv:`INSTANCE` - the name of your |Gel| Cloud instance (``/``)\n\n\nTwo ways to use Gel Cloud\n=========================\n\n1. CLI\n^^^^^^\n\nLog in to |Gel| Cloud via the CLI:\n\n.. code-block:: bash\n\n $ gel cloud login\n\n\nThis will open a browser window and allow you to log in via GitHub.\nNow, create your |Gel| Cloud instance the same way you would create a\nlocal instance:\n\n.. code-block:: bash\n\n $ gel instance create /\n\nor\n\n.. code-block:: bash\n\n $ gel project init \\\n --server-instance /\n\n\n2. GUI\n^^^^^^\n\nCreate your instance at `cloud.geldata.com `_ by\nclicking on ā€œCreate new instanceā€ in the ā€œInstancesā€ tab.\n\n..
\n\nComplete the following form to configure your instance. You can access\nyour instance via the CLI using the name ``/`` or via the GUI.\n\n\nUseful Gel Cloud commands\n=========================\n\nGet REPL\n^^^^^^^^\n\n.. code-block:: bash\n\n $ gel \\\n -I /\n\nRun migrations\n^^^^^^^^^^^^^^\n\n.. code-block:: bash\n\n $ gel migrate \\\n -I /\n\nUpdate your instance\n^^^^^^^^^^^^^^^^^^^^\n\n.. code-block:: bash\n\n $ gel instance upgrade \\\n --to-version \\\n -I /\n\nManual full backup\n^^^^^^^^^^^^^^^^^^\n\n.. code-block:: bash\n\n $ gel dump \\\n --all --format dir \\\n -I / \\\n \n\nFull restore\n^^^^^^^^^^^^\n\n.. code-block:: bash\n\n $ gel restore \\\n --all \\\n -I / \\\n \n\n.. note::\n\n Restoring works only to an empty database.\n\n\nQuestions? Problems? Bugs?\n==========================\n\nThank you for helping us make the best way to host your |Gel| instances even\nbetter!\n\n* Please join us on `our Discord `_ to ask\n questions.\n* If you're experiencing a service interruption, check `our status page\n `_ for information on what may be\n causing it.\n* Report any bugs you find by `submitting a support ticket\n `_. Note: when using |Gel| Cloud\n through the CLI, setting the ``RUST_LOG`` environment variable to ``info``,\n ``debug``, or ``trace`` may provide additional debugging information\n which will be useful to include with your ticket.\n" }, { "path": "docs/cloud/migrate_from.rst", "content": ".. _ref_migrate_from:\n\n=======================================\nMigrating from Gel Cloud to Self-Hosted\n=======================================\n\n:edb-alt-title: Migrating from Gel Cloud to Self-Hosted Gel\n\n|Gel| Cloud is sunsetting at the end of January 2026. To ensure your\napplications continue to run smoothly, you should migrate your data to a\nself-hosted |Gel| instance.\n\nThis guide outlines the process of migrating your production data. We strongly\nrecommend performing a \"dry run\" with your staging or development environment\nfirst to familiarize yourself with the workflow.\n\n\nPhase 1: Preparation\n====================\n\n1. Spin up your self-hosted deployment\n--------------------------------------\n\nWhile you can host |Gel| on any infrastructure that supports Docker or binary\ninstallations, we recommend using a managed cloud provider (such as Fly.io,\nAWS, or GCP with managed Postgres) for production reliability.\n\nSee our :ref:`self-hosted deployment guides ` for\nstep-by-step instructions on deploying to various platforms.\n\nEnsure your new instance is:\n\n* Running the same version of |Gel| as your Cloud instance (or newer).\n* Configured with a persistent volume for data or connected to a managed\n Postgres instance.\n\n2. Retrieve connection parameters\n---------------------------------\n\nOnce your new instance is live, you need its DSN (Data Source Name) or\nindividual connection parameters. Each :ref:`deployment guide\n` outlines the best way to retrieve the various\nconnection parameters from your specific setup. You will typically need:\n\n* **Host**: The domain or IP of your new instance.\n* **Port**: Default is ``5656``.\n* **User**: Default is |admin|.\n* **Password**: The password you set during initialization.\n* **TLS CA**: The certificate used to secure the connection (unless using a\n public CA or ``--trust-tls-cert``).\n\nThe DSN format is:\n\n.. code-block:: text\n\n gel://:@:/\n\nAll components except the scheme are optional. See :ref:`ref_reference_connection_dsn`\nfor more details.\n\n\nPhase 2: The Migration (Cutover)\n================================\n\nTo ensure data consistency, you must prevent new writes to your database\nduring the transfer.\n\n1. Enable maintenance mode\n--------------------------\n\nBefore touching the data, put your application into maintenance mode. This\nensures that no new records are created in |Gel| Cloud after you've started\nthe dump.\n\n* **Web Apps**: Point your load balancer to a static \"Maintenance\" page.\n* **Background Jobs**: Stop all workers, cron jobs, or queues that interact\n with the database.\n\n2. Perform the migration\n------------------------\n\nYou can use the |Gel| CLI to move data directly from your Cloud instance to\nyour new self-hosted instance.\n\n.. code-block:: bash\n\n # 1. Dump from Gel Cloud to directory\n $ gel dump --instance / \\\n --all --format=dir \\\n production_dump\n\n # 2. Restore to self-hosted from dump\n $ gel restore --dsn --all production_dump\n\n.. note::\n\n If your self-hosted instance uses a self-signed TLS certificate, you may\n need to add ``--tls-security insecure`` to the restore command, or first\n retrieve the TLS certificate and set it via :gelenv:`TLS_CA`.\n\n\nPhase 3: Verification and Go-Live\n=================================\n\n1. Update application environment variables\n-------------------------------------------\n\nUpdate your application's configuration to point to the new instance. Replace\nthe Gel Cloud specific connection environment variables :gelenv:`INSTANCE` and\n:gelenv:`SECRET_KEY` variables with the new connection details:\n\n* :gelenv:`DSN`: :geluri:`user:password@host:port/branch`\n* :gelenv:`TLS_CA`: The TLS certificate content (if your instance uses a\n self-signed certificate)\n* :gelenv:`CLIENT_TLS_SECURITY`: Set to ``insecure`` if you need to skip TLS\n verification (not recommended for production)\n\n2. Sanity check\n---------------\n\nBefore turning off maintenance mode:\n\n* Run a few :gelcmd:`query` commands against the new instance to verify data\n integrity.\n* Check that your schema migrated correctly: :gelcmd:`migrate --status`.\n* Launch a local instance of your app connected to the new production DB to\n ensure connection logic is sound.\n\n3. Disable maintenance mode\n---------------------------\n\nOnce verified, restart your application servers and background workers.\nMonitor your logs closely for any connection or permission errors.\n\n\nPost-Migration Note\n===================\n\nOnce you are 100% certain your data is safe and your app is stable on the new\nhost, you can de-provision your |Gel| Cloud instance. Remember that all |Gel|\nCloud data will be deleted after the January 2026 deadline.\n" }, { "path": "docs/cloud/web.rst", "content": ".. _ref_guide_cloud_web:\n\n=======\nWeb GUI\n=======\n\n:edb-alt-title: Using Gel Cloud via the web GUI\n\nIf you'd prefer, you can also manage your account via `the Gel Cloud\nweb-based GUI `_.\n\nThe first time you access the web UI, you will be prompted to log in. Once you\nlog in with your account, you'll be on the \"Instances\" tab of the front page\nwhich shows your instance list. The other two tabs allow you to manage your\norganization settings and billing.\n\nInstances\n---------\n\nIf this is your first time accessing Gel Cloud, this list will be empty. To\ncreate an instance, click \"Create new instance.\" This will pop up a modal\nallowing you to name your instance and specify the version of Gel and the\nregion for the instance.\n\nOnce the instance has been created, you'll see the instance dashboard which\nallows you to monitor your instance, navigate to the management page for its\nbranches, and create secret keys.\n\nYou'll also see instructions in the bottom-right for linking your |Gel| CLI to\nyour Gel Cloud account. You do this by running the CLI command\n:gelcmd:`cloud login`. This will make all of your Gel Cloud instances accessible via\nthe CLI. You can manage them just as you would other remote Gel instances.\n\nIf you want to manage a branch of your database, click through on the\ninstance's name from the top right of the instance dashboard. If you just\ncreated a database, the branch management view will be mostly empty except\nfor a button offering to create a sample branch. Once you have a schema\ncreated and some data in a database, this view will offer you similar tools to\nthose in our local UI.\n\nYou'll be able to access a REPL, edit complex queries or build them\ngraphically, inspect your schema, and browse your data.\n\nOrg Settings\n------------\n\nThis tab allows you to add GitHub organizations for which you are an admin.\nIf you don't see your organization's name here, you may need to update your\n`org settings`_ in GitHub to allow |Gel| Cloud to read your list of\norganizations, and then refresh the org list.\n\n.. lint-off\n\n.. _org setings:\n https://docs.github.com/en/organizations/managing-oauth-access-to-your-organizations-data/approving-oauth-apps-for-your-organization\n\n.. lint-on\n\nBilling\n-------\n\nOn this page you can manage your account type and payment methods, and set your\nemail for receiving billing info. Optionally, you can also save your payment\ninfo using `Link `_, `Stripe's `_\nfast-checkout solution.\n" }, { "path": "docs/conf.py", "content": "# -*- coding: utf-8 -*-\n#\n# EdgeDB documentation build configuration file, created by\n# sphinx-quickstart on Wed Aug 3 17:58:14 2016.\n#\n# This file is execfile()d with the current directory set to its\n# containing dir.\n#\n# Note that not all possible configuration values are present in this\n# autogenerated file.\n#\n# All configuration values have a default; values that are commented out\n# serve to show the default.\n\n\n# If extensions (or modules to document with autodoc) are in another directory,\n# add these directories to sys.path here. If the directory is relative to the\n# documentation root, use os.path.abspath to make it absolute, like shown here.\n# sys.path.insert(0, os.path.abspath('.'))\n\n# -- General configuration ------------------------------------------------\n\n\n# If your documentation needs a minimal Sphinx version, state it here.\n# needs_sphinx = '1.0'\n\n# Add any Sphinx extension module names here, as strings. They can be\n# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom\n# ones.\nextensions = [\n 'sphinx.ext.autodoc',\n 'sphinx.ext.todo',\n 'sphinx.ext.viewcode',\n 'edb.tools.docs',\n 'sphinxcontrib.asyncio',\n 'sphinx.ext.intersphinx',\n 'sphinx_code_tabs',\n]\n\nintersphinx_mapping = {'python': ('https://docs.python.org/3', None)}\n\n# Add any paths that contain templates here, relative to this directory.\ntemplates_path = []\n\n# The suffix(es) of source filenames.\n# You can specify multiple suffix as a list of string:\n# source_suffix = ['.rst', '.md']\nsource_suffix = '.rst'\n\n# The encoding of source files.\n# source_encoding = 'utf-8-sig'\n\n# The master toctree document.\nmaster_doc = 'index'\n\n# General information about the project.\nproject = u'EdgeDB'\ncopyright = u'2016, magicstack'\nauthor = u'magicstack'\n\n# The version info for the project you're documenting, acts as replacement for\n# |version| and |release|, also used in various other places throughout the\n# built documents.\n#\n# The short X.Y version.\nversion = '0.5.0'\n# The full version, including alpha/beta/rc tags.\nrelease = '0.5.0'\n\n# The language for content autogenerated by Sphinx. Refer to documentation\n# for a list of supported languages.\n#\n# This is also used if you do content translation via gettext catalogs.\n# Usually you set \"language\" from the command line for these cases.\nlanguage = None\n\n# There are two options for replacing |today|: either, you set today to some\n# non-false value, then it is used:\n# today = ''\n# Else, today_fmt is used as the format for a strftime call.\n# today_fmt = '%B %d, %Y'\n\n# List of patterns, relative to source directory, that match files and\n# directories to ignore when looking for source files.\nexclude_patterns = []\n\n# The reST default role (used for this markup: `text`) to use for all\n# documents.\n# default_role = None\n\n# If true, '()' will be appended to :func: etc. cross-reference text.\n# add_function_parentheses = True\n\n# If true, the current module name will be prepended to all description\n# unit titles (such as .. function::).\n# add_module_names = True\n\n# If true, sectionauthor and moduleauthor directives will be shown in the\n# output. They are ignored by default.\n# show_authors = False\n\n# The name of the Pygments (syntax highlighting) style to use.\npygments_style = 'sphinx'\n\n# A list of ignored prefixes for module index sorting.\n# modindex_common_prefix = []\n\n# If true, keep warnings as \"system message\" paragraphs in the built documents.\n# keep_warnings = False\n\nsuppress_warnings = ['image.not_readable']\n\n# If true, `todo` and `todoList` produce output, else they produce nothing.\ntodo_include_todos = False\n\nprimary_domain = None\n\n\n# -- Options for HTML output ----------------------------------------------\n\n# The theme to use for HTML and HTML Help pages. See the documentation for\n# a list of builtin themes.\nhtml_theme = 'alabaster'\n\n# Theme options are theme-specific and customize the look and feel of a theme\n# further. For a list of options available for each theme, see the\n# documentation.\n# html_theme_options = {}\n\n# Add any paths that contain custom themes here, relative to this directory.\n# html_theme_path = []\n\n# The name for this set of Sphinx documents. If None, it defaults to\n# \" v documentation\".\n# html_title = None\n\n# A shorter title for the navigation bar. Default is the same as html_title.\n# html_short_title = None\n\n# The name of an image file (relative to this directory) to place at the top\n# of the sidebar.\n# html_logo = None\n\n# The name of an image file (within the static path) to use as favicon of the\n# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32\n# pixels large.\n# html_favicon = None\n\n# Add any paths that contain custom static files (such as style sheets) here,\n# relative to this directory. They are copied after the builtin static files,\n# so a file named \"default.css\" will overwrite the builtin \"default.css\".\nhtml_static_path = []\n\n# Add any extra paths that contain custom files (such as robots.txt or\n# .htaccess) here, relative to this directory. These files are copied\n# directly to the root of the documentation.\n# html_extra_path = []\n\n# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,\n# using the given strftime format.\n# html_last_updated_fmt = '%b %d, %Y'\n\n# If true, SmartyPants will be used to convert quotes and dashes to\n# typographically correct entities.\n# html_use_smartypants = True\n\n# Custom sidebar templates, maps document names to template names.\nhtml_sidebars = {\n '**': [\n 'globaltoc.html',\n 'searchbox.html',\n ]\n}\n\n# Additional templates that should be rendered to pages, maps page names to\n# template names.\n# html_additional_pages = {}\n\n# If false, no module index is generated.\n# html_domain_indices = True\n\n# If false, no index is generated.\n# html_use_index = True\n\n# If true, the index is split into individual pages for each letter.\n# html_split_index = False\n\n# If true, links to the reST sources are added to the pages.\n# html_show_sourcelink = True\n\n# If true, \"Created using Sphinx\" is shown in the HTML footer. Default is True.\n# html_show_sphinx = True\n\n# If true, \"(C) Copyright ...\" is shown in the HTML footer. Default is True.\n# html_show_copyright = True\n\n# If true, an OpenSearch description file will be output, and all pages will\n# contain a tag referring to it. The value of this option must be the\n# base URL from which the finished HTML is served.\n# html_use_opensearch = ''\n\n# This is the file name suffix for HTML files (e.g. \".xhtml\").\n# html_file_suffix = None\n\n# Language to be used for generating the HTML full-text search index.\n# Sphinx supports the following languages:\n# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'\n# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'\n# html_search_language = 'en'\n\n# A dictionary with options for the search language support, empty by default.\n# Now only 'ja' uses this config value\n# html_search_options = {'type': 'default'}\n\n# The name of a javascript file (relative to the configuration directory) that\n# implements a search results scorer. If empty, the default will be used.\n# html_search_scorer = 'scorer.js'\n\n# Output file base name for HTML help builder.\nhtmlhelp_basename = 'EdgeDBdoc'\n\n# -- Options for LaTeX output ---------------------------------------------\n\nlatex_elements = {\n # The paper size ('letterpaper' or 'a4paper').\n # 'papersize': 'letterpaper',\n\n # The font size ('10pt', '11pt' or '12pt').\n # 'pointsize': '10pt',\n\n # Additional stuff for the LaTeX preamble.\n # 'preamble': '',\n\n # Latex figure (float) alignment\n # 'figure_align': 'htbp',\n}\n\n# Grouping the document tree into LaTeX files. List of tuples\n# (source start file, target name, title,\n# author, documentclass [howto, manual, or own class]).\nlatex_documents = [\n (master_doc, 'EdgeDB.tex', u'EdgeDB Documentation',\n u'magicstack', 'manual'),\n]\n\n# The name of an image file (relative to this directory) to place at the top of\n# the title page.\n# latex_logo = None\n\n# For \"manual\" documents, if this is true, then toplevel headings are parts,\n# not chapters.\n# latex_use_parts = False\n\n# If true, show page references after internal links.\n# latex_show_pagerefs = False\n\n# If true, show URL addresses after external links.\n# latex_show_urls = False\n\n# Documents to append as an appendix to all manuals.\n# latex_appendices = []\n\n# If false, no module index is generated.\n# latex_domain_indices = True\n\n\n# -- Options for manual page output ---------------------------------------\n\n# One entry per manual page. List of tuples\n# (source start file, name, description, authors, manual section).\nman_pages = [\n (master_doc, 'edgedb', u'EdgeDB Documentation',\n [author], 1)\n]\n\n# If true, show URL addresses after external links.\n# man_show_urls = False\n\n\n# -- Options for Texinfo output -------------------------------------------\n\n# Grouping the document tree into Texinfo files. List of tuples\n# (source start file, target name, title, author,\n# dir menu entry, description, category)\ntexinfo_documents = [\n (master_doc, 'EdgeDB', u'EdgeDB Documentation',\n author, 'EdgeDB', 'One line description of project.',\n 'Miscellaneous'),\n]\n\n# Documents to append as an appendix to all manuals.\n# texinfo_appendices = []\n\n# If false, no module index is generated.\n# texinfo_domain_indices = True\n\n# How to display URL addresses: 'footnote', 'no', or 'inline'.\n# texinfo_show_urls = 'footnote'\n\n# If true, do not generate a @detailmenu in the \"Top\" node's menu.\n# texinfo_no_detailmenu = False\n\n\n# config for srclink\n\nsrclink_project = 'https://github.com/edgedb/edgedb'\nsrclink_src_path = 'doc/'\nsrclink_branch = 'doc'\n" }, { "path": "docs/index.rst", "content": ".. _index_toplevel:\n\n\n=================\nGel documentation\n=================\n\nWelcome to the |Gel| |version| documentation.\n\n\n.. !!! DO NOT CHANGE :maxdepth: egdedb.com/docs depends on it !!!\n.. toctree::\n :maxdepth: 5\n :includehidden:\n\n intro/index\n reference/index\n resources/index\n cloud/index\n" }, { "path": "docs/intro/branches.rst", "content": ".. _ref_intro_branches:\n\n========\nBranches\n========\n\n|Gel's| branches make it easy to prototype app features that impact your\ndatabase schema, even in cases where those features are never released. You can\ncreate a branch in your Gel database that corresponds to a feature branch in\nyour VCS. When you're done, either :ref:`merge `\nthat branch into your main branch or :ref:`drop `\nit leaving your original schema intact.\n\n.. note::\n\n The procedure we will describe should be adaptable to any VCS offering\n branching and rebasing, but in order to make the examples concrete and\n easy-to-follow, we'll be demonstrating how Gel branches interact with\n Git branches. You may adapt these examples to your VCS of choice.\n\n\n1. Create a new feature branch\n------------------------------\n\nCreate a feature branch in your VCS and switch to it. Then, create and switch\nto a corresponding branch in Gel using the CLI.\n\n.. code-block:: bash\n\n $ gel branch create feature\n Creating branch 'feature'...\n OK: CREATE BRANCH\n $ gel branch switch feature\n Switching from 'main' to 'feature'\n\n.. note::\n\n You can alternatively create and switch in one shot using :gelcmd:`branch\n switch -c feature`.\n\n\n2. Build your feature\n---------------------\n\nWrite your code and make any schema changes your feature requires.\n\n\n3. Pull any changes on \"main\"\n-----------------------------\n\n.. note::\n\n This step is optional. If you know your |main| code branch is current and\n all migrations in that code branch have already been applied to your\n |main| database branch, feel free to skip it.\n\nWe need to make sure that merging our feature branch onto |main| is a simple\nfast-forward. The next two steps take care of that.\n\nSwitch back to your |main| code branch. Run ``git pull`` to pull down any new\nchanges. If any of these are schema changes, use :gelcmd:`branch switch main`\nto switch back to your |main| database branch and apply the new schema with\n:gelcmd:`migrate`.\n\nOnce this is done, you can switch back to your feature branches in your VCS and\n|Gel|.\n\n\n4. Rebase your feature branch on \"main\"\n---------------------------------------\n\n.. note::\n\n If you skipped the previous step, you can skip this one too. This is only\n necessary if you had to pull down new changes on |main|.\n\nFor your code branch, first make sure you're on ``feature`` and then run the\nrebase:\n\n.. code-block:: bash\n\n $ git rebase main\n\nNow, do the same for your database, also from ``feature``:\n\n.. code-block:: bash\n\n $ gel branch rebase main\n\n\n5. Merge ``feature`` onto \"main\"\n--------------------------------\n\nSwitch back to both |main| branches and merge ``feature``.\n\n.. code-block:: bash\n\n $ git switch main\n \n Switched to branch 'main'\n $ git merge feature\n\n.. code-block:: bash\n\n $ gel branch switch main\n Switching from 'feature' to 'main'\n $ gel branch merge feature\n\nNow, your feature and its schema have been successfully merged! šŸŽ‰\n\n\nFurther reading\n^^^^^^^^^^^^^^^\n\n- :ref:`Branches CLI `\n\nFurther information can be found in the `branches RFC\n`_,\nwhich describes the design of the migration system.\n" }, { "path": "docs/intro/cli.rst", "content": ".. _ref_intro_cli:\n\n.. _ref_admin_install:\n\n=======\nThe CLI\n=======\n\nThe |gelcmd| command line tool is an integral part of the developer workflow\nof building with Gel. Below are instructions for installing it.\n\nInstallation\n------------\n\nTo get started with Gel, the first step is install the |gelcmd| CLI.\n\n**Linux or macOS**\n\n.. code-block:: bash\n\n $ curl --proto '=https' --tlsv1.2 -sSf https://www.geldata.com/sh | sh\n\n**Windows Powershell**\n\n.. note::\n\n Gel on Windows requires WSL 2 because the Gel server runs on Linux.\n\n.. code-block:: powershell\n\n PS> iwr https://www.geldata.com/ps1 -useb | iex\n\nFollow the prompts on screen to complete the installation. The script will\ndownload the |gelcmd| command built for your OS and add a path to it to your\nshell environment. Then test the installation:\n\n.. code-block:: bash\n\n $ gel --version\n Gel CLI x.x+abcdefg\n\n.. note::\n\n If you encounter a ``command not found`` error, you may need to open a fresh\n shell window.\n\n\nSee ``help`` commands\n---------------------\n\nThe entire CLI is self-documenting. Once it's installed, run :gelcmd:`--help`\nto see a breakdown of all the commands and options.\n\n.. code-block:: bash\n\n $ gel --help\n Usage: gel [OPTIONS] [COMMAND]\n\n Commands:\n \n\n Options:\n \n\n Connection Options (gel --help-connect to see full list):\n \n\n Cloud Connection Options:\n \n\nThe majority of CLI commands perform some action against a *particular* Gel\ninstance. As such, there are a standard set of flags that are used to specify\n*which instance* should be the target of the command, plus additional\ninformation like TLS certificates. The following command documents these flags.\n\n.. code-block:: bash\n\n $ gel --help-connect\n Connection Options (full list):\n\n -I, --instance \n Instance name (use `gel instance list` to list local, remote and\n Cloud instances available to you)\n\n --dsn \n DSN for Gel to connect to (overrides all other options except\n password)\n\n --credentials-file \n Path to JSON file to read credentials from\n\n -H, --host \n Gel instance host\n\n -P, --port \n Port to connect to Gel\n\n --unix-path \n A path to a Unix socket for Gel connection\n\n When the supplied path is a directory, the actual path will be\n computed using the `--port` and `--admin` parameters.\n ...\n\nIf you ever want to see documentation for a particular command (\n:gelcmd:`migration create`) or group of commands (:gelcmd:`instance`),\njust append the ``--help`` flag.\n\n.. code-block:: bash\n\n $ gel instance --help\n Manage local Gel instances\n\n Usage: gel instance \n\n Commands:\n create Initialize a new Gel instance\n list Show all instances\n status Show status of an instance\n start Start an instance\n stop Stop an instance\n ...\n\nUpgrade the CLI\n---------------\n\nTo upgrade to the latest version:\n\n.. code-block:: bash\n\n $ gel cli upgrade\n" }, { "path": "docs/intro/clients.rst", "content": ".. _ref_intro_clients:\n\n================\nClient Libraries\n================\n\n|Gel| implements libraries for popular languages that make it easier to work\nwith Gel. These libraries provide a common set of functionality.\n\n- *Instantiating clients.* Most libraries implement a ``Client`` class that\n internally manages a pool of physical connections to your Gel instance.\n- *Resolving connections.* All client libraries implement a standard protocol\n for determining how to connect to your database. In most cases, this will\n involve checking for special environment variables like :gelenv:`DSN` or, in\n the case of Gel Cloud instances, :gelenv:`INSTANCE` and\n :gelenv:`SECRET_KEY`.\n (More on this in :ref:`the Connection section below\n `.)\n- *Executing queries.* A ``Client`` will provide some methods for executing\n queries against your database. Under the hood, this query is executed using\n Gel's efficient binary protocol.\n\n.. note::\n\n For some use cases, you may not need a client library. Gel allows you to\n execute :ref:`queries over HTTP `. This is slower than the\n binary protocol and lacks support for transactions and rich data types, but\n may be suitable if a client library isn't available for your language of\n choice.\n\nAvailable libraries\n===================\n\nTo execute queries from your application code, use one of :ref:`Gel's client\nlibraries `.\n\nUsage\n=====\n\nTo follow along with the guide below, first create a new directory and\ninitialize a project.\n\n.. code-block:: bash\n\n $ mydir myproject\n $ cd myproject\n $ gel project init\n\nConfigure the environment as needed for your preferred language.\n\n.. tabs::\n\n .. code-tab:: bash\n :caption: Node.js\n\n $ npm init -y\n $ tsc --init # (TypeScript only)\n $ touch index.ts\n\n .. code-tab:: bash\n :caption: Deno\n\n $ touch index.ts\n\n .. code-tab:: bash\n :caption: Python\n\n $ python -m venv venv\n $ source venv/bin/activate\n $ touch main.py\n\n .. code-tab:: bash\n :caption: Rust\n\n $ cargo init\n\n .. code-tab:: bash\n :caption: Go\n\n $ go mod init example/quickstart\n $ touch hello.go\n\n .. code-tab:: bash\n :caption: .NET\n\n $ dotnet new console -o . -f net6.0\n\n\nInstall the Gel client library.\n\n.. tabs::\n\n .. code-tab:: bash\n :caption: Node.js\n\n $ npm install gel # npm\n $ yarn add gel # yarn\n\n .. code-tab:: txt\n :caption: Deno\n\n n/a\n\n .. code-tab:: bash\n :caption: Python\n\n $ pip install gel\n\n .. code-tab:: toml\n :caption: Rust\n\n # Cargo.toml\n\n [dependencies]\n gel-tokio = \"0.5.0\"\n # Additional dependency\n tokio = { version = \"1.28.1\", features = [\"macros\", \"rt-multi-thread\"] }\n\n .. code-tab:: bash\n :caption: Go\n\n $ go get github.com/geldata/gel-go\n\n .. code-tab:: bash\n :caption: .NET\n\n $ dotnet add package Gel.Net.Driver\n\n\nCopy and paste the following simple script. This script initializes a\n``Client`` instance. Clients manage an internal pool of connections to your\ndatabase and provide a set of methods for executing queries.\n\n.. note::\n\n Note that we aren't passing connection information (say, a connection\n URL) when creating a client. The client libraries can detect that\n they are inside a project directory and connect to the project-linked\n instance automatically. For details on configuring connections, refer\n to the :ref:`Connection ` section below.\n\n.. lint-off\n\n.. tabs::\n\n .. code-tab:: typescript\n :caption: Node.js\n\n import {createClient} from 'gel';\n\n const client = createClient();\n\n client.querySingle(`select random()`).then((result) => {\n console.log(result);\n });\n\n\n .. code-tab:: python\n\n from gel import create_client\n\n client = create_client()\n\n result = client.query_single(\"select random()\")\n print(result)\n\n .. code-tab:: rust\n\n // src/main.rs\n #[tokio::main]\n async fn main() {\n let conn = gel_tokio::create_client()\n .await\n .expect(\"Client initiation\");\n let val = conn\n .query_required_single::(\"select random()\", &())\n .await\n .expect(\"Returning value\");\n println!(\"Result: {}\", val);\n }\n\n .. code-tab:: go\n\n // hello.go\n package main\n\n import (\n \"context\"\n \"fmt\"\n \"log\"\n\n \"github.com/geldata/gel-go\"\n )\n\n func main() {\n ctx := context.Background()\n client, err := gel.CreateClient(ctx, gel.Options{})\n if err != nil {\n log.Fatal(err)\n }\n defer client.Close()\n\n var result float64\n err = client.\n QuerySingle(ctx, \"select random();\", &result)\n if err != nil {\n log.Fatal(err)\n }\n\n fmt.Println(result)\n }\n\n .. code-tab:: csharp\n :caption: .NET\n\n using Gel;\n\n var client = new GelClient();\n var result = await client.QuerySingleAsync(\"select random();\");\n Console.WriteLine(result);\n\n .. code-tab:: elixir\n :caption: Elixir\n\n # lib/gel_quickstart.ex\n defmodule GelQuickstart do\n def run do\n {:ok, client} = Gel.start_link()\n result = Gel.query_single!(client, \"select random()\")\n IO.inspect(result)\n end\n end\n\n.. lint-on\n\n\nFinally, execute the file.\n\n.. tabs::\n\n .. code-tab:: bash\n :caption: Node.js\n\n $ npx tsx index.ts\n\n .. code-tab:: bash\n :caption: Deno\n\n $ deno run --allow-all --unstable index.deno.ts\n\n .. code-tab:: bash\n :caption: Python\n\n $ python index.py\n\n .. code-tab:: bash\n :caption: Rust\n\n $ cargo run\n\n .. code-tab:: bash\n :caption: Go\n\n $ go run .\n\n .. code-tab:: bash\n :caption: .NET\n\n $ dotnet run\n\n .. code-tab:: bash\n :caption: Elixir\n\n $ mix run -e GelQuickstart.run\n\nYou should see a random number get printed to the console. This number was\ngenerated inside your Gel instance using EdgeQL's built-in\n:eql:func:`random` function.\n\n.. _ref_intro_clients_connection:\n\nConnection\n==========\n\nAll client libraries implement a standard protocol for determining how to\nconnect to your database.\n\nUsing projects\n--------------\n\nIn development, we recommend :ref:`initializing a\nproject ` in the root of your codebase.\n\n.. code-block:: bash\n\n $ gel project init\n\nOnce the project is initialized, any code that uses an official client library\nwill automatically connect to the project-linked instanceā€”no need for\nenvironment variables or hard-coded credentials. Follow the :ref:`Using\nprojects ` guide to get started.\n\nUsing environment variables\n---------------------------\n\n.. _ref_intro_clients_connection_cloud:\n\nFor Gel Cloud\n^^^^^^^^^^^^^\n\nIn production, connection information can be securely passed to the client\nlibrary via environment variables. For Gel Cloud instances, the recommended\nvariables to set are :gelenv:`INSTANCE` and :gelenv:`SECRET_KEY`.\n\nSet :gelenv:`INSTANCE` to ``/`` where\n```` is the name you set when you created the Gel Cloud\ninstance.\n\nIf you have not yet created a secret key, you can do so in the Gel Cloud UI\nor by running :ref:`ref_cli_gel_cloud_secretkey_create` via the CLI.\n\nFor self-hosted instances\n^^^^^^^^^^^^^^^^^^^^^^^^^\n\nMost commonly for self-hosted remote instances, you set a value for the\n:gelenv:`DSN` environment variable.\n\n.. note::\n\n If environment variables like :gelenv:`DSN` are defined inside a project\n directory, the environment variables will take precedence.\n\nA DSN is also known as a \"connection string\" and takes the\nfollowing form: :geluri:`:@:`.\n\n\nEach element of the DSN is optional; in fact |geluri| is a technically a\nvalid DSN. Any unspecified element will default to the following values.\n\n.. list-table::\n\n * - ````\n - ``localhost``\n * - ````\n - ``5656``\n * - ````\n - |admin|\n * - ````\n - ``null``\n\nA typical DSN may look like this:\n:geluri:`admin:PASSWORD@db.domain.com:8080`.\n\nDSNs can also contain the following query parameters.\n\n.. list-table::\n\n * - ``branch``\n - The database branch to connect to within the given instance. Defaults to\n |main|.\n\n * - ``tls_security``\n - The TLS security mode. Accepts the following values.\n\n - ``\"strict\"`` (**default**) ā€” verify certificates and hostnames\n - ``\"no_host_verification\"`` ā€” verify certificates only\n - ``\"insecure\"`` ā€” trust self-signed certificates\n\n * - ``tls_ca_file``\n - A filesystem path pointing to a CA root certificate. This is usually only\n necessary when attempting to connect via TLS to a remote instance with a\n self-signed certificate.\n\nThese parameters can be added to any DSN using web-standard query string\nnotation: :geluri:`user:pass@example.com:8080?branch=my_branch&tls_security=insecure`.\n\n\nFor a more comprehensive guide to DSNs, see the :ref:`DSN Specification\n`.\n\nUsing multiple environment variables\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nIf needed for your deployment pipeline, each element of the DSN can be\nspecified independently.\n\n- :gelenv:`HOST`\n- :gelenv:`PORT`\n- :gelenv:`USER`\n- :gelenv:`PASSWORD`\n- :gelenv:`BRANCH`\n- :gelenv:`TLS_CA_FILE`\n- :gelenv:`CLIENT_TLS_SECURITY`\n\n.. note::\n\n If a value for :gelenv:`DSN` is defined, it will override these variables!\n\nOther mechanisms\n----------------\n\n:gelenv:`CREDENTIALS_FILE`\n A path to a ``.json`` file containing connection information. In some\n scenarios (including local Docker development) its useful to represent\n connection information with files.\n\n .. code-block:: json\n\n {\n \"host\": \"localhost\",\n \"port\": 10700,\n \"user\": \"testuser\",\n \"password\": \"testpassword\",\n \"branch\": \"main\",\n \"tls_cert_data\": \"-----BEGIN CERTIFICATE-----\\nabcdef...\"\n }\n\n:gelenv:`INSTANCE` (local/Gel Cloud only)\n The name of an instance. Useful only for local or Gel Cloud instances.\n\n .. note::\n\n For more on Gel Cloud instances, see the :ref:`Gel Cloud instance\n connection section ` above.\n\nReference\n---------\n\nThese are the most common ways to connect to an instance, however Gel\nsupports several other options for advanced use cases. For a complete reference\non connection configuration, see :ref:`Reference > Connection Parameters\n`.\n" }, { "path": "docs/intro/edgeql.rst", "content": ".. _ref_intro_edgeql:\n\nEdgeQL\n======\n\nEdgeQL is the query language of Gel. It's intended as a spiritual successor\nto SQL that solves some of its biggest design limitations. This page is\nintended as a rapid-fire overview so you can hit the ground running with\n|Gel|. Refer to the linked pages for more in-depth documentation.\n\n\n.. note::\n\n The examples below also demonstrate how to express the query with the\n :ref:`TypeScript client's ` query builder, which lets you\n express arbitrary EdgeQL queries in a code-first, typesafe way.\n\n\nScalar literals\n^^^^^^^^^^^^^^^\n\n|Gel| has a rich primitive type system consisting of the following data types.\n\n.. list-table::\n\n * - Strings\n - ``str``\n * - Booleans\n - ``bool``\n * - Numbers\n - ``int16`` ``int32`` ``int64`` ``float32`` ``float64``\n ``bigint`` ``decimal``\n * - UUID\n - ``uuid``\n * - JSON\n - ``json``\n * - Dates and times\n - ``datetime`` ``cal::local_datetime`` ``cal::local_date``\n ``cal::local_time``\n * - Durations\n - ``duration`` ``cal::relative_duration`` ``cal::date_duration``\n * - Binary data\n - ``bytes``\n * - Auto-incrementing counters\n - ``sequence``\n * - Enums\n - ``enum``\n\nBasic literals can be declared using familiar syntax.\n\n.. tabs::\n\n .. code-tab:: edgeql-repl\n\n db> select \"I ā¤ļø EdgeQL\"; # str\n {'U ā¤ļø EdgeQL'}\n db> select false; # bool\n {false}\n db> select 42; # int64\n {42}\n db> select 3.14; # float64\n {3.14}\n db> select 12345678n; # bigint\n {12345678n}\n db> select 15.0e+100n; # decimal\n {15.0e+100n}\n db> select b'bina\\\\x01ry'; # bytes\n {b'bina\\\\x01ry'}\n\n\n .. code-tab:: typescript\n\n e.str(\"I ā¤ļø EdgeQL\")\n // string\n e.bool(false)\n // boolean\n e.int64(42)\n // number\n e.float64(3.14)\n // number\n e.bigint(BigInt(12345678))\n // bigint\n e.decimal(\"1234.4567\")\n // n/a (not supported by JS clients)\n e.bytes(Buffer.from(\"bina\\\\x01ry\"))\n // Buffer\n\nOther type literals are declared by *casting* an appropriately\nstructured string.\n\n.. tabs::\n\n .. code-tab:: edgeql-repl\n\n db> select 'a5ea6360-75bd-4c20-b69c-8f317b0d2857';\n {a5ea6360-75bd-4c20-b69c-8f317b0d2857}\n db> select '1999-03-31T15:17:00Z';\n {'1999-03-31T15:17:00Z'}\n db> select '5 hours 4 minutes 3 seconds';\n {'5:04:03'}\n db> select '2 years 18 days';\n {'P2Y18D'}\n\n\n .. code-tab:: typescript\n\n e.uuid(\"a5ea6360-75bd-4c20-b69c-8f317b0d2857\")\n // string\n e.datetime(\"1999-03-31T15:17:00Z\")\n // Date\n e.duration(\"5 hours 4 minutes 3 seconds\")\n // gel.Duration (custom class)\n e.cal.relative_duration(\"2 years 18 days\")\n // gel.RelativeDuration (custom class)\n\nPrimitive data can be composed into arrays and tuples, which can themselves be\nnested.\n\n.. tabs::\n\n .. code-tab:: edgeql-repl\n\n db> select ['hello', 'world'];\n {['hello', 'world']}\n db> select ('Apple', 7, true);\n {('Apple', 7, true)} # unnamed tuple\n db> select (fruit := 'Apple', quantity := 3.14, fresh := true);\n {(fruit := 'Apple', quantity := 3.14, fresh := true)} # named tuple\n db> select [\"this\", \"is\", \"an\", \"array\"];\n {\"[\\\"this\\\", \\\"is\\\", \\\"an\\\", \\\"array\\\"]\"}\n\n .. code-tab:: typescript\n\n e.array([\"hello\", \"world\"]);\n // string[]\n e.tuple([\"Apple\", 7, true]);\n // [string, number, boolean]\n e.tuple({fruit: \"Apple\", quantity: 3.14, fresh: true});\n // {fruit: string; quantity: number; fresh: boolean}\n e.json([\"this\", \"is\", \"an\", \"array\"]);\n // unknown\n\n\n|Gel| also supports a special ``json`` type for representing unstructured\ndata. Primitive data structures can be converted to JSON using a type cast\n(````). Alternatively, a properly JSON-encoded string can be converted\nto ``json`` with the built-in ``to_json`` function. Indexing a ``json`` value\nreturns another ``json`` value.\n\n.. code-tabs::\n\n .. code-tab:: edgeql-repl\n\n gel> select 5;\n {\"5\"}\n gel> select [1,2,3];\n {\"[1, 2, 3]\"}\n gel> select to_json('[{ \"name\": \"Peter Parker\" }]');\n {\"[{\\\"name\\\": \\\"Peter Parker\\\"}]\"}\n gel> select to_json('[{ \"name\": \"Peter Parker\" }]')[0]['name'];\n {\"\\\"Peter Parker\\\"\"}\n\n .. code-tab:: typescript\n\n /*\n The result of an query returning `json` is represented\n with `unknown` in TypeScript.\n */\n e.json(5); // => unknown\n e.json([1, 2, 3]); // => unknown\n e.to_json('[{ \"name\": \"Peter Parker\" }]'); // => unknown\n e.to_json('[{ \"name\": \"Peter Parker\" }]')[0][\"name\"]; // => unknown\n\n\nRefer to :ref:`Docs > EdgeQL > Literals ` for complete docs.\n\nFunctions and operators\n^^^^^^^^^^^^^^^^^^^^^^^\n\n|Gel| provides a rich standard library of functions to operate and manipulate\nvarious data types.\n\n.. tabs::\n\n .. code-tab:: edgeql-repl\n\n db> select str_upper('oh hi mark');\n {'OH HI MARK'}\n db> select len('oh hi mark');\n {10}\n db> select uuid_generate_v1mc();\n {c68e3836-0d59-11ed-9379-fb98e50038bb}\n db> select contains(['a', 'b', 'c'], 'd');\n {false}\n\n .. code-tab:: typescript\n\n e.str_upper(\"oh hi mark\");\n // string\n e.len(\"oh hi mark\");\n // number\n e.uuid_generate_v1mc();\n // string\n e.contains([\"a\", \"b\", \"c\"], \"d\");\n // boolean\n\nSimilarly, it provides a comprehensive set of built-in operators.\n\n.. tabs::\n\n .. code-tab:: edgeql-repl\n\n db> select not true;\n {false}\n db> select exists 'hi';\n {true}\n db> select 2 + 2;\n {4}\n db> select 'Hello' ++ ' world!';\n {'Hello world!'}\n db> select 'šŸ˜„' if true else 'šŸ˜¢';\n {'šŸ˜„'}\n db> select '5 minutes' + '2 hours';\n {'2:05:00'}\n\n .. code-tab:: typescript\n\n e.op(\"not\", e.bool(true));\n // booolean\n e.op(\"exists\", e.set(\"hi\"));\n // boolean\n e.op(\"exists\", e.cast(e.str, e.set()));\n // boolean\n e.op(e.int64(2), \"+\", e.int64(2));\n // number\n e.op(e.str(\"Hello \"), \"++\", e.str(\"World!\"));\n // string\n e.op(e.str(\"šŸ˜„\"), \"if\", e.bool(true), \"else\", e.str(\"šŸ˜¢\"));\n // string\n e.op(e.duration(\"5 minutes\"), \"+\", e.duration(\"2 hours\"))\n\nSee :ref:`Docs > Standard Library ` for reference documentation on\nall built-in types, including the functions and operators that apply to them.\n\nInsert an object\n^^^^^^^^^^^^^^^^\n\nObjects are created using ``insert``. The ``insert`` statement relies on\ndeveloper-friendly syntax like curly braces and the ``:=`` operator.\n\n.. tabs::\n\n .. code-tab:: edgeql\n\n insert Movie {\n title := 'Doctor Strange 2',\n release_year := 2022\n };\n\n .. code-tab:: typescript\n\n const query = e.insert(e.Movie, {\n title: 'Doctor Strange 2',\n release_year: 2022\n });\n\n const result = await query.run(client);\n // {id: string}\n // by default INSERT only returns\n // the id of the new object\n\nSee :ref:`Docs > EdgeQL > Insert `.\n\nNested inserts\n^^^^^^^^^^^^^^\n\nOne of EdgeQL's greatest features is that it's easy to compose. Nested inserts\nare easily achieved with subqueries.\n\n.. tabs::\n\n .. code-tab:: edgeql\n\n insert Movie {\n title := 'Doctor Strange 2',\n release_year := 2022,\n director := (insert Person {\n name := 'Sam Raimi'\n })\n };\n\n .. code-tab:: typescript\n\n const query = e.insert(e.Movie, {\n title: 'Doctor Strange 2',\n release_year: 2022,\n director: e.insert(e.Person, {\n name: 'Sam Raimi'\n })\n });\n\n const result = await query.run(client);\n // {id: string}\n // by default INSERT only returns\n // the id of the new object\n\nSelect objects\n^^^^^^^^^^^^^^\n\nUse a *shape* to define which properties to ``select`` from the given object\ntype.\n\n.. tabs::\n\n .. code-tab:: edgeql\n\n select Movie {\n id,\n title\n };\n\n .. code-tab:: typescript\n\n const query = e.select(e.Movie, () => ({\n id: true,\n title: true\n }));\n const result = await query.run(client);\n // {id: string; title: string; }[]\n\n // To select all properties of an object, use the\n // spread operator with the special \"*\"\" property:\n const query = e.select(e.Movie, () => ({\n ...e.Movie['*']\n }));\n\nFetch linked objects with a nested shape.\n\n.. tabs::\n\n .. code-tab:: edgeql\n\n select Movie {\n id,\n title,\n actors: {\n name\n }\n };\n\n .. code-tab:: typescript\n\n const query = e.select(e.Movie, () => ({\n id: true,\n title: true,\n actors: {\n name: true,\n }\n }));\n\n const result = await query.run(client);\n // {id: string; title: string, actors: {name: string}[]}[]\n\nSee :ref:`Docs > EdgeQL > Select > Shapes `.\n\nFiltering, ordering, and pagination\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nThe ``select`` statement can be augmented with ``filter``, ``order by``,\n``offset``, and ``limit`` clauses (in that order).\n\n.. tabs::\n\n .. code-tab:: edgeql\n\n select Movie {\n id,\n title\n }\n filter .release_year > 2017\n order by .title\n offset 10\n limit 10;\n\n .. code-tab:: typescript\n\n const query = e.select(e.Movie, (movie) => ({\n id: true,\n title: true,\n filter: e.op(movie.release_year, \">\", 1999),\n order_by: movie.title,\n offset: 10,\n limit: 10,\n }));\n\n const result = await query.run(client);\n // {id: string; title: number}[]\n\nNote that you reference properties of the object to include in your ``select``\nby prepending the property name with a period: ``.release_year``. This is known\nas *leading dot notation*.\n\nEvery new set of curly braces introduces a new scope. You can add ``filter``,\n``limit``, and ``offset`` clauses to nested shapes.\n\n.. tabs::\n\n .. code-tab:: edgeql\n\n select Movie {\n title,\n actors: {\n name\n } filter .name ilike 'chris%'\n }\n filter .title ilike '%avengers%';\n\n .. code-tab:: typescript\n\n e.select(e.Movie, movie => ({\n title: true,\n characters: c => ({\n name: true,\n filter: e.op(c.name, \"ilike\", \"chris%\"),\n }),\n filter: e.op(movie.title, \"ilike\", \"%avengers%\"),\n }));\n // => { characters: { name: string; }[]; title: string; }[]\n\n const result = await query.run(client);\n // {id: string; title: number}[]\n\n\n\nSee :ref:`Filtering `, :ref:`Ordering\n`, and :ref:`Pagination `.\n\nQuery composition\n^^^^^^^^^^^^^^^^^\n\nWe've seen how to ``insert`` and ``select``. How do we do both in one query?\nAnswer: query composition. EdgeQL's syntax is designed to be *composable*,\nlike any good programming language.\n\n.. tabs::\n\n .. code-tab:: edgeql\n\n select (\n insert Movie { title := 'The Marvels' }\n ) {\n id,\n title\n };\n\n .. code-tab:: typescript\n\n const newMovie = e.insert(e.Movie, {\n title: \"The Marvels\"\n });\n const query = e.select(newMovie, () => ({\n id: true,\n title: true\n }));\n\n const result = await query.run(client);\n // {id: string; title: string}\n\nWe can clean up this query by pulling out the ``insert`` statement into a\n``with`` block. A ``with`` block is useful for composing complex multi-step\nqueries, like a script.\n\n.. tabs::\n\n .. code-tab:: edgeql\n\n with new_movie := (insert Movie { title := 'The Marvels' })\n select new_movie {\n id,\n title\n };\n\n .. code-tab:: typescript\n\n /*\n Same as above.\n\n In the query builder, explicit ``with`` blocks aren't necessary!\n Just assign your EdgeQL subqueries to variables and compose them as you\n like. The query builder automatically convert your top-level query to an\n EdgeQL expression with proper ``with`` blocks.\n */\n\nComputed properties\n^^^^^^^^^^^^^^^^^^^\n\nSelection shapes can contain computed properties.\n\n.. tabs::\n\n .. code-tab:: edgeql\n\n select Movie {\n title,\n title_upper := str_upper(.title),\n cast_size := count(.actors)\n };\n\n .. code-tab:: typescript\n\n e.select(e.Movie, movie => ({\n title: true,\n title_upper: e.str_upper(movie.title),\n cast_size: e.count(movie.actors)\n }))\n // {title: string; title_upper: string; cast_size: number}[]\n\nA common use for computed properties is to query a link in reverse; this is\nknown as a *backlink* and it has special syntax.\n\n.. tabs::\n\n .. code-tab:: edgeql\n\n select Person {\n name,\n acted_in := . ({\n name: true,\n acted_in: e.select(person[\" ({\n title: true,\n })),\n }));\n // {name: string; acted_in: {title: string}[];}[]\n\nSee :ref:`Docs > EdgeQL > Select > Computed fields ` and\n:ref:`Docs > EdgeQL > Select > Backlinks `.\n\nUpdate objects\n^^^^^^^^^^^^^^\n\nThe ``update`` statement accepts a ``filter`` clause up-front, followed by a\n``set`` shape indicating how the matching objects should be updated.\n\n.. tabs::\n\n .. code-tab:: edgeql\n\n update Movie\n filter .title = \"Doctor Strange 2\"\n set {\n title := \"Doctor Strange in the Multiverse of Madness\"\n };\n\n .. code-tab:: typescript\n\n const query = e.update(e.Movie, (movie) => ({\n filter: e.op(movie.title, '=', 'Doctor Strange 2'),\n set: {\n title: 'Doctor Strange in the Multiverse of Madness',\n },\n }));\n\n const result = await query.run(client);\n // {id: string}\n\nWhen updating links, the set of linked objects can be added to with ``+=``,\nsubtracted from with ``-=``, or overwritten with ``:=``.\n\n.. tabs::\n\n .. code-tab:: edgeql\n\n update Movie\n filter .title = \"Doctor Strange 2\"\n set {\n actors += (select Person filter .name = \"Rachel McAdams\")\n };\n\n .. code-tab:: typescript\n\n e.update(e.Movie, (movie) => ({\n filter: e.op(movie.title, '=', 'Doctor Strange 2'),\n set: {\n actors: {\n \"+=\": e.select(e.Person, person => ({\n filter: e.op(person.name, \"=\", \"Rachel McAdams\")\n }))\n }\n },\n }));\n\nSee :ref:`Docs > EdgeQL > Update `.\n\nDelete objects\n^^^^^^^^^^^^^^\n\nThe ``delete`` statement can contain ``filter``, ``order by``, ``offset``, and\n``limit`` clauses.\n\n.. tabs::\n\n .. code-tab:: edgeql\n\n delete Movie\n filter .title ilike \"the avengers%\"\n limit 3;\n\n .. code-tab:: typescript\n\n const query = e.delete(e.Movie, (movie) => ({\n filter: e.op(movie.title, 'ilike', \"the avengers%\"),\n }));\n\n const result = await query.run(client);\n // {id: string}[]\n\nSee :ref:`Docs > EdgeQL > Delete `.\n\nQuery parameters\n^^^^^^^^^^^^^^^^\n\nYou can reference query parameters in your queries with ``$`` notation.\nSince EdgeQL is a strongly typed language, all query parameters must be\nprepending with a *type cast* to indicate the expected type.\n\n.. note::\n\n Scalars like ``str``, ``int64``, and ``json`` are\n supported. Tuples, arrays, and object types are not.\n\n.. tabs::\n\n .. code-tab:: edgeql\n\n insert Movie {\n title := $title,\n release_year := $release_year\n };\n\n .. code-tab:: typescript\n\n const query = e.params({ title: e.str, release_year: e.int64 }, ($) => {\n return e.insert(e.Movie, {\n title: $.title,\n release_year: $.release_year,\n }))\n };\n\n const result = await query.run(client, {\n title: 'Thor: Love and Thunder',\n release_year: 2022,\n });\n // {id: string}\n\nAll client libraries provide a dedicated API for specifying parameters when\nexecuting a query.\n\n.. tabs::\n\n .. code-tab:: javascript\n\n import {createClient} from \"gel\";\n\n const client = createClient();\n const result = await client.query(`select $param`, {\n param: \"Play it, Sam.\"\n });\n // => \"Play it, Sam.\"\n\n .. code-tab:: python\n\n import gel\n\n client = gel.create_async_client()\n\n async def main():\n result = await client.query(\"select $param\", param=\"Play it, Sam\")\n # => \"Play it, Sam\"\n\n .. code-tab:: go\n\n package main\n\n import (\n \"context\"\n \"log\"\n\n \"github.com/geldata/gel-go\"\n )\n\n func main() {\n ctx := context.Background()\n client, err := gel.CreateClient(ctx, gel.Options{})\n if err != nil {\n log.Fatal(err)\n }\n defer client.Close()\n\n var (\n param string = \"Play it, Sam.\"\n result string\n )\n\n query := \"select $0\"\n err = client.Query(ctx, query, &result, param)\n // ...\n }\n\n .. code-tab:: rust\n\n // [dependencies]\n // gel-tokio = \"0.5.0\"\n // tokio = { version = \"1.28.1\", features = [\"macros\", \"rt-multi-thread\"] }\n\n #[tokio::main]\n async fn main() {\n let conn = gel_tokio::create_client()\n .await\n .expect(\"Client initiation\");\n let param = \"Play it, Sam.\";\n let val = conn\n .query_required_single::(\"select $0\", &(param,))\n .await\n .expect(\"Returning value\");\n println!(\"{val}\");\n }\n\nSee :ref:`Docs > EdgeQL > Parameters `.\n\nSubqueries\n^^^^^^^^^^\n\nUnlike SQL, EdgeQL is *composable*; queries can be naturally nested. This is\nuseful, for instance, when performing nested mutations.\n\n.. tabs::\n\n .. code-tab:: edgeql\n\n with\n dr_strange := (select Movie filter .title = \"Doctor Strange\"),\n benedicts := (select Person filter .name in {\n 'Benedict Cumberbatch',\n 'Benedict Wong'\n })\n update dr_strange\n set {\n actors += benedicts\n };\n\n .. code-tab:: typescript\n\n // select Doctor Strange\n const drStrange = e.select(e.Movie, movie => ({\n filter: e.op(movie.title, '=', \"Doctor Strange\")\n }));\n\n // select actors\n const actors = e.select(e.Person, person => ({\n filter: e.op(person.name, 'in', e.set(\n 'Benedict Cumberbatch',\n 'Benedict Wong'\n ))\n }));\n\n // add actors to cast of drStrange\n const query = e.update(drStrange, ()=>({\n actors: { \"+=\": actors }\n }));\n\nWe can also use subqueries to fetch properties of an object we just inserted.\n\n.. tabs::\n\n .. code-tab:: edgeql\n\n with new_movie := (insert Movie {\n title := \"Avengers: The Kang Dynasty\",\n release_year := 2025\n })\n select new_movie {\n title, release_year\n };\n\n .. code-tab:: typescript\n\n // \"with\" blocks are added automatically\n // in the generated query!\n\n const newMovie = e.insert(e.Movie, {\n title: \"Avengers: The Kang Dynasty\",\n release_year: 2025\n });\n\n const query = e.select(newMovie, ()=>({\n title: true,\n release_year: true,\n }));\n\n const result = await query.run(client);\n // {title: string; release_year: number;}\n\nSee :ref:`Docs > EdgeQL > Select > Subqueries `.\n\nPolymorphic queries\n^^^^^^^^^^^^^^^^^^^\n\nConsider the following schema.\n\n.. code-block:: sdl\n\n abstract type Content {\n required title: str;\n }\n\n type Movie extending Content {\n release_year: int64;\n }\n\n type TVShow extending Content {\n num_seasons: int64;\n }\n\nWe can ``select`` the abstract type ``Content`` to simultaneously fetch all\nobjects that extend it, and use the ``[is ]`` syntax to select\nproperties from known subtypes.\n\n.. tabs::\n\n .. code-tab:: edgeql\n\n select Content {\n title,\n [is TVShow].num_seasons,\n [is Movie].release_year\n };\n\n .. code-tab:: typescript\n\n const query = e.select(e.Content, (content) => ({\n title: true,\n ...e.is(e.Movie, {release_year: true}),\n ...e.is(e.TVShow, {num_seasons: true}),\n }));\n /* {\n title: string;\n release_year: number | null;\n num_seasons: number | null;\n }[] */\n\nSee :ref:`Docs > EdgeQL > Select > Polymorphic queries\n`.\n\nGrouping objects\n^^^^^^^^^^^^^^^^\n\nUnlike SQL, EdgeQL provides a top-level ``group`` statement to compute\ngroupings of objects.\n\n.. tabs::\n\n .. code-tab:: edgeql\n\n group Movie { title, actors: { name }}\n by .release_year;\n\n .. code-tab:: typescript\n\n e.group(e.Movie, (movie) => {\n const release_year = movie.release_year;\n return {\n title: true,\n by: {release_year},\n };\n });\n /* {\n grouping: string[];\n key: { release_year: number | null };\n elements: { title: string; }[];\n }[] */\n\nSee :ref:`Docs > EdgeQL > Group `.\n" }, { "path": "docs/intro/guides/ai/edgeql.rst", "content": ".. _ref_ai_guide_edgeql:\n\n================\nGel AI in EdgeQL\n================\n\n:edb-alt-title: How to set up Gel AI in EdgeQL\n\n\n|Gel| AI brings vector search capabilities and retrieval-augmented generation\ndirectly into the database.\n\n\nEnable and configure the extension\n==================================\n\n.. edb:split-section::\n\n AI is a |Gel| extension. To enable it, we will need to add the extension\n to the appā€™s schema:\n\n .. code-block:: sdl\n\n using extension ai;\n\n\n.. edb:split-section::\n\n |Gel| AI uses external APIs in order to get vectors and LLM completions. For it\n to work, we need to configure an API provider and specify their API key. Let's\n open EdgeQL REPL and run the following query:\n\n .. code-block:: edgeql\n\n configure current database\n insert ext::ai::OpenAIProviderConfig {\n secret := 'sk-....',\n };\n\n\nNow our |Gel| application can take advantage of OpenAI's API to implement AI\ncapabilities.\n\n\n.. note::\n\n |Gel| AI comes with its own :ref:`UI ` that can\n be used to configure providers, set up prompts and test them in a sandbox.\n\n\n.. note::\n\n Most API providers require you to set up and account and charge money for\n model use.\n\n\nAdd vectors and perform similarity search\n=========================================\n\n.. edb:split-section::\n\n Before we start introducing AI capabilities, let's set up our database with a\n schema and populate it with some data (we're going to be helping Komi-san keep\n track of her friends).\n\n .. code-block:: sdl\n\n module default {\n type Friend {\n required name: str {\n constraint exclusive;\n };\n\n summary: str; # A brief description of personality and role\n relationship_to_komi: str; # Relationship with Komi\n defining_trait: str; # Primary character trait or quirk\n }\n }\n\n.. edb:split-section::\n\n Here's a shell command you can paste and run that will populate the\n database with some sample data.\n\n .. code-block:: bash\n :class: collapsible\n\n $ cat << 'EOF' > populate_db.edgeql\n insert Friend {\n name := 'Tadano Hitohito',\n summary := 'An extremely average high school boy with a remarkable ability to read the atmosphere and understand others\\' feelings, especially Komi\\'s.',\n relationship_to_komi := 'First friend and love interest',\n defining_trait := 'Perceptiveness',\n };\n\n insert Friend {\n name := 'Osana Najimi',\n summary := 'An extremely outgoing person who claims to have been everyone\\'s childhood friend. Gender: Najimi.',\n relationship_to_komi := 'Second friend and social catalyst',\n defining_trait := 'Universal childhood friend',\n };\n\n insert Friend {\n name := 'Yamai Ren',\n summary := 'An intense and sometimes obsessive classmate who is completely infatuated with Komi.',\n relationship_to_komi := 'Self-proclaimed guardian and admirer',\n defining_trait := 'Obsessive devotion',\n };\n\n insert Friend {\n name := 'Katai Makoto',\n summary := 'A intimidating-looking but shy student who shares many communication problems with Komi.',\n relationship_to_komi := 'Fellow communication-challenged friend',\n defining_trait := 'Scary appearance but gentle nature',\n };\n\n insert Friend {\n name := 'Nakanaka Omoharu',\n summary := 'A self-proclaimed wielder of dark powers who acts like an anime character and is actually just a regular gaming enthusiast.',\n relationship_to_komi := 'Gaming buddy and chuunibyou friend',\n defining_trait := 'Chuunibyou tendencies',\n };\n EOF\n $ gel query -f populate_db.edgeql\n\n\n.. edb:split-section::\n\n In order to get |Gel| to produce embedding vectors, we need to create a special\n ``deferred index`` on the type we would like to perform similarity search on.\n More specifically, we need to specify an EdgeQL expression that produces a\n string that we're going to create an embedding vector for. This is how we would\n set up an index if we wanted to perform similarity search on\n ``Friend.summary``:\n\n .. code-block:: sdl-diff\n\n module default {\n type Friend {\n required name: str {\n constraint exclusive;\n };\n\n summary: str; # A brief description of personality and role\n relationship_to_komi: str; # Relationship with Komi\n defining_trait: str; # Primary character trait or quirk\n\n + deferred index ext::ai::index(embedding_model := 'text-embedding-3-small')\n + on (.summary);\n }\n }\n\n\n.. edb:split-section::\n\n But actually, in our case it would be better if we could similarity search\n across all properties at the same time. We can define the index on a more\n complex expression - like a concatenation of string properties - like this:\n\n\n .. code-block:: sdl-diff\n\n module default {\n type Friend {\n required name: str {\n constraint exclusive;\n };\n\n summary: str; # A brief description of personality and role\n relationship_to_komi: str; # Relationship with Komi\n defining_trait: str; # Primary character trait or quirk\n\n deferred index ext::ai::index(embedding_model := 'text-embedding-3-small')\n - on (.summary);\n + on (\n + .name ++ ' ' ++ .summary ++ ' '\n + ++ .relationship_to_komi ++ ' '\n + ++ .defining_trait\n + );\n }\n }\n\n\n.. edb:split-section::\n\n Once we're done with schema modification, we need to apply them by going\n through a migration:\n\n .. code-block:: bash\n\n $ gel migration create\n $ gel migrate\n\n\n.. edb:split-section::\n\n That's it! |Gel| will make necessary API requests in the background and create an\n index that will enable us to perform efficient similarity search like this:\n\n .. code-block:: edgeql\n\n select ext::ai::search(Friend, query_vector);\n\n\n.. edb:split-section::\n\n Note that this function accepts an embedding vector as the second argument, not\n a text string. This means that in order to similarity search for a string, we\n need to create a vector embedding for it using the same model as we used to\n create the index. |Gel| offers an HTTP endpoint ``/ai/embeddings`` that can\n handle it for us. All we need to do is to pass the vector it produces into the\n search query:\n\n .. note::\n\n Note that we're passing our login and password in order to autheticate the\n request. We can find those using the CLI: :gelcmd:`instance credentials\n --json`. Learn about all the other ways you can authenticate a request\n :ref:`here `.\n\n .. code-block:: bash\n\n $ curl --user user:password \\\n --json '{\"input\": \"Who helps Komi make friends?\", \"model\": \"text-embedding-3-small\"}' \\\n http://localhost:/branch/main/ai/embeddings \\\n | jq -r '.data[0].embedding' \\ # extract the embedding out of the JSON\n | tr -d '\\n' \\ # remove newlines\n | sed 's/^\\[//;s/\\]$//' \\ # remove square brackets\n | awk '{print \"select ext::ai::search(Friend, >[\" $0 \"]);\"}' \\ # assemble the query\n | gel query --file - # pass the query into Gel CLI\n\n\n\nUse the built-in RAG\n====================\n\nOne more feature |Gel| AI offers is built-in retrieval-augmented generation, also\nknown as RAG.\n\n.. edb:split-section::\n\n |Gel| comes preconfigured to be able to process our text query, perform\n similarity search across the index we just created, pass the results to an LLM\n and return a response. We can access the built-in RAG using the ``/ai/rag``\n HTTP endpoint:\n\n\n .. code-block:: bash\n\n $ curl --user user:password --json '{\n \"query\": \"Who helps Komi make friends?\",\n \"model\": \"gpt-4-turbo-preview\",\n \"context\": {\"query\":\"select Friend\"}\n }' http://localhost:/branch/main/ai/rag\n\n\n.. edb:split-section::\n\n We can also stream the response like this:\n\n\n .. code-block:: bash-diff\n\n $ curl --user user:password --json '{\n \"query\": \"Who helps Komi make friends?\",\n \"model\": \"gpt-4-turbo-preview\",\n \"context\": {\"query\":\"select Friend\"},\n + \"stream\": true,\n }' http://localhost:/branch/main/ai/rag\n\n\nKeep going!\n===========\n\nYou are now sufficiently equipped to use |Gel| AI in your applications.\n\nIf you'd like to build something on your own, make sure to check out the\n:ref:`Reference manual ` in order to learn the details\nabout using different APIs and models, configuring prompts or using the UI.\nMake sure to also check out the |Gel| AI bindings in :ref:`Python\n` and :ref:`JavaScript `\nif those languages are relevant to you.\n\nAnd if you would like more guidance for how |Gel| AI can be fit into an\napplication, take a look at the :ref:`FastAPI Gel AI Tutorial\n`, where we're building a search bot using\nfeatures you learned about above.\n\n" }, { "path": "docs/intro/guides/ai/index.rst", "content": ".. edb:env-switcher::\n\n=========\nAdding AI\n=========\n\n.. toctree::\n :maxdepth: 1\n :hidden:\n\n edgeql\n python" }, { "path": "docs/intro/guides/ai/python.rst", "content": ".. _ref_ai_guide_python:\n\n================\nGel AI in Python\n================\n\n:edb-alt-title: How to set up Gel AI in Python\n\n.. edb:split-section::\n\n |Gel| AI brings vector search capabilities and retrieval-augmented\n generation directly into the database. It's integrated into the |Gel|\n Python binding via the ``gel.ai`` module.\n\n .. code-block:: bash\n\n $ pip install 'gel[ai]'\n\n\nEnable and configure the extension\n==================================\n\n.. edb:split-section::\n\n AI is an |Gel| extension. To enable it, we will need to add the extension\n to the appā€™s schema:\n\n .. code-block:: sdl\n\n using extension ai;\n\n\n.. edb:split-section::\n\n |Gel| AI uses external APIs in order to get vectors and LLM completions.\n For it to work, we need to configure an API provider and specify their API\n key. Let's open EdgeQL REPL and run the following query:\n\n .. code-block:: edgeql\n\n configure current database\n insert ext::ai::OpenAIProviderConfig {\n secret := 'sk-....',\n };\n\n\nNow our |Gel| application can take advantage of OpenAI's API to implement AI\ncapabilities.\n\n\n.. note::\n\n |Gel| AI comes with its own :ref:`UI ` that can\n be used to configure providers, set up prompts and test them in a sandbox.\n\n\n.. note::\n\n Most API providers require you to set up and account and charge money for\n model use.\n\n\nAdd vectors\n===========\n\n.. edb:split-section::\n\n Before we start introducing AI capabilities, let's set up our database with a\n schema and populate it with some data (we're going to be helping Komi-san keep\n track of her friends).\n\n .. code-block:: sdl\n\n module default {\n type Friend {\n required name: str {\n constraint exclusive;\n };\n\n summary: str; # A brief description of personality and role\n relationship_to_komi: str; # Relationship with Komi\n defining_trait: str; # Primary character trait or quirk\n }\n }\n\n.. edb:split-section::\n\n Here's a shell command you can paste and run that will populate the\n database with some sample data.\n\n .. code-block:: bash\n :class: collapsible\n\n $ cat << 'EOF' > populate_db.edgeql\n insert Friend {\n name := 'Tadano Hitohito',\n summary := 'An extremely average high school boy with a remarkable ability to read the atmosphere and understand others\\' feelings, especially Komi\\'s.',\n relationship_to_komi := 'First friend and love interest',\n defining_trait := 'Perceptiveness',\n };\n\n insert Friend {\n name := 'Osana Najimi',\n summary := 'An extremely outgoing person who claims to have been everyone\\'s childhood friend. Gender: Najimi.',\n relationship_to_komi := 'Second friend and social catalyst',\n defining_trait := 'Universal childhood friend',\n };\n\n insert Friend {\n name := 'Yamai Ren',\n summary := 'An intense and sometimes obsessive classmate who is completely infatuated with Komi.',\n relationship_to_komi := 'Self-proclaimed guardian and admirer',\n defining_trait := 'Obsessive devotion',\n };\n\n insert Friend {\n name := 'Katai Makoto',\n summary := 'A intimidating-looking but shy student who shares many communication problems with Komi.',\n relationship_to_komi := 'Fellow communication-challenged friend',\n defining_trait := 'Scary appearance but gentle nature',\n };\n\n insert Friend {\n name := 'Nakanaka Omoharu',\n summary := 'A self-proclaimed wielder of dark powers who acts like an anime character and is actually just a regular gaming enthusiast.',\n relationship_to_komi := 'Gaming buddy and chuunibyou friend',\n defining_trait := 'Chuunibyou tendencies',\n };\n EOF\n $ gel query -f populate_db.edgeql\n\n\n.. edb:split-section::\n\n In order to get |Gel| to produce embedding vectors, we need to create a\n special ``deferred index`` on the type we would like to perform similarity\n search on. More specifically, we need to specify an EdgeQL expression that\n produces a string that we're going to create an embedding vector for. This\n is how we would set up an index if we wanted to perform similarity search\n on ``Friend.summary``:\n\n .. code-block:: sdl-diff\n\n module default {\n type Friend {\n required name: str {\n constraint exclusive;\n };\n\n summary: str; # A brief description of personality and role\n relationship_to_komi: str; # Relationship with Komi\n defining_trait: str; # Primary character trait or quirk\n\n + deferred index ext::ai::index(embedding_model := 'text-embedding-3-small')\n + on (.summary);\n }\n }\n\n\n.. edb:split-section::\n\n But actually, in our case it would be better if we could similarity search\n across all properties at the same time. We can define the index on a more\n complex expression - like a concatenation of string properties - like this:\n\n\n .. code-block:: sdl-diff\n\n module default {\n type Friend {\n required name: str {\n constraint exclusive;\n };\n\n summary: str; # A brief description of personality and role\n relationship_to_komi: str; # Relationship with Komi\n defining_trait: str; # Primary character trait or quirk\n\n deferred index ext::ai::index(embedding_model := 'text-embedding-3-small')\n - on (.summary);\n + on (\n + .name ++ ' ' ++ .summary ++ ' '\n + ++ .relationship_to_komi ++ ' '\n + ++ .defining_trait\n + );\n }\n }\n\n\n.. edb:split-section::\n\n Once we're done with schema modification, we need to apply them by going\n through a migration:\n\n .. code-block:: bash\n\n $ gel migration create\n $ gel migrate\n\n\nThat's it! |Gel| will make necessary API requests in the background and create an\nindex that will enable us to perform efficient similarity search.\n\n\nPerform similarity search in Python\n===================================\n\n.. edb:split-section::\n\n In order to run queries against the index we just created, we need to create a\n |Gel| client and pass it to a |Gel| AI instance.\n\n .. code-block:: python\n\n import gel\n import gel.ai\n\n gel_client = gel.create_client()\n gel_ai = gel.ai.create_rag_client(client)\n\n text = \"Who helps Komi make friends?\"\n vector = gel_ai.generate_embeddings(\n text,\n \"text-embedding-3-small\",\n )\n\n gel_client.query(\n \"select ext::ai::search(Friend, >$embedding_vector\",\n embedding_vector=vector,\n )\n\n\n.. edb:split-section::\n\n We are going to execute a query that calls a single function:\n ``ext::ai::search(, )``. That function accepts an\n embedding vector as the second argument, not a text string. This means that in\n order to similarity search for a string, we need to create a vector embedding\n for it using the same model as we used to create the index. The |Gel| AI binding\n in Python comes with a ``generate_embeddings`` function that does exactly that:\n\n\n .. code-block:: python-diff\n\n import gel\n import gel.ai\n\n gel_client = gel.create_client()\n gel_ai = gel.ai.create_rag_client(client)\n\n + text = \"Who helps Komi make friends?\"\n + vector = gel_ai.generate_embeddings(\n + text,\n + \"text-embedding-3-small\",\n + )\n\n\n.. edb:split-section::\n\n Now we can plug that vector directly into our query to get similarity search\n results:\n\n\n .. code-block:: python-diff\n\n import gel\n import gel.ai\n\n gel_client = gel.create_client()\n gel_ai = gel.ai.create_rag_client(client)\n\n text = \"Who helps Komi make friends?\"\n vector = gel_ai.generate_embeddings(\n text,\n \"text-embedding-3-small\",\n )\n\n + gel_client.query(\n + \"select ext::ai::search(Friend, >$embedding_vector\",\n + embedding_vector=vector,\n + )\n\n\nUse the built-in RAG\n====================\n\nOne more feature |Gel| AI offers is built-in retrieval-augmented generation,\nalso known as RAG.\n\n.. edb:split-section::\n\n |Gel| comes preconfigured to be able to process our text query, perform\n similarity search across the index we just created, pass the results to an\n LLM and return a response. In order to access the built-in RAG, we need to\n start by selecting an LLM and passing its name to the |Gel| AI instance\n constructor:\n\n\n .. code-block:: python-diff\n\n import gel\n import gel.ai\n\n gel_client = gel.create_client()\n gel_ai = gel.ai.create_rag_client(\n client,\n + model=\"gpt-4-turbo-preview\"\n )\n\n\n.. edb:split-section::\n\n Now we can access the RAG using the ``query_rag`` function like this:\n\n\n .. code-block:: python-diff\n\n import gel\n import gel.ai\n\n gel_client = gel.create_client()\n gel_ai = gel.ai.create_rag_client(\n client,\n model=\"gpt-4-turbo-preview\"\n )\n\n + gel_ai.query_rag(\n + \"Who helps Komi make friends?\",\n + context=\"Friend\",\n + )\n\n\n.. edb:split-section::\n\n We can also stream the response like this:\n\n\n .. code-block:: python-diff\n\n import gel\n import gel.ai\n\n gel_client = gel.create_client()\n gel_ai = gel.ai.create_rag_client(\n client,\n model=\"gpt-4-turbo-preview\"\n )\n\n - gel_ai.query_rag(\n + gel_ai.stream_rag(\n \"Who helps Komi make friends?\",\n context=\"Friend\",\n )\n\n\nKeep going!\n===========\n\nYou are now sufficiently equipped to use |Gel| AI in your applications.\n\nIf you'd like to build something on your own, make sure to check out the\n:ref:`Reference manual ` for the AI extension in order\nto learn the details about using different APIs and models, configuring prompts\nor using the UI. Make sure to take a look at the :ref:`Python binding reference\n`, too.\n\nAnd if you would like more guidance for how |Gel| AI can be fit into an\napplication, take a look at the :ref:`FastAPI Gel AI Tutorial\n`, where we're building a search bot using\nfeatures you learned about above.\n\n\n" }, { "path": "docs/intro/guides/drizzle/index.rst", "content": ".. edb:env-switcher::\n\n==================\nAdding Drizzle ORM\n==================\n\n.. toctree::\n :maxdepth: 1\n :hidden:\n\n nextjs\n" }, { "path": "docs/intro/guides/drizzle/nextjs.rst", "content": ".. _ref_guide_gel_drizzle:\n\n======================\nDrizzle ORM in Next.js\n======================\n\n|Gel| integrates seamlessly with Drizzle ORM, providing a type-safe and intuitive way to interact with your database in TypeScript applications.\n\nEnable Drizzle in your Gel project\n==================================\n\n.. edb:split-section::\n\n To integrate Drizzle with your Gel project, you'll need to install the necessary dependencies:\n\n .. code-block:: bash\n\n $ npm install drizzle-orm\n $ npm install -D drizzle-kit\n\n\n.. edb:split-section::\n\n Next, create a Drizzle configuration file in your project root to tell Drizzle how to work with your Gel database:\n\n .. code-block:: typescript\n :caption: drizzle.config.ts\n\n import { defineConfig } from 'drizzle-kit';\n\n export default defineConfig({\n dialect: 'gel',\n });\n\n\nSync your Gel schema with Drizzle\n=================================\n\n.. edb:split-section::\n\n Before using Drizzle with your Gel database, you'll need to let Drizzle introspect your schema. This step generates TypeScript files that Drizzle can use to interact with your database.\n\n .. code-block:: bash\n\n $ npx drizzle-kit pull\n\n\n.. edb:split-section::\n\n This command will create a schema file based on your Gel database. The file will typically look something like this:\n\n .. code-block:: typescript\n :caption: drizzle/schema.ts\n :class: collapsible\n\n import { gelTable, uniqueIndex, uuid, smallint, text, timestamp, relations } from \"drizzle-orm/gel-core\"\n import { sql } from \"drizzle-orm\"\n\n export const books = gelTable(\"Book\", {\n id: uuid().default(sql`uuid_generate_v4()`).primaryKey().notNull(),\n title: text().notNull(),\n author: text(),\n year: smallint(),\n genre: text(),\n read_date: timestamp(),\n }, (table) => [\n uniqueIndex(\"books_pkey\").using(\"btree\", table.id.asc().nullsLast().op(\"uuid_ops\")),\n ]);\n\n export const notes = gelTable(\"Note\", {\n id: uuid().default(sql`uuid_generate_v4()`).primaryKey().notNull(),\n text: text().notNull(),\n created_at: timestamp().default(sql`datetime_current()`),\n book_id: uuid().notNull(),\n }, (table) => [\n uniqueIndex(\"notes_pkey\").using(\"btree\", table.id.asc().nullsLast().op(\"uuid_ops\")),\n ]);\n\n\nKeep Drizzle in sync with Gel\n=============================\n\n.. edb:split-section::\n\n To keep your Drizzle schema in sync with your Gel schema, add a hook to your ``gel.toml`` file. This hook will automatically run ``drizzle-kit pull`` after each migration:\n\n .. code-block:: toml\n :caption: gel.toml\n\n [hooks]\n after_migration_apply = [\n \"npx drizzle-kit pull\"\n ]\n\n\n\nWith this hook in place, your Drizzle schema will automatically update whenever you apply Gel migrations.\n\n\nCreate a database client\n========================\n\n.. edb:split-section::\n\n Now, let's create a database client that you can use throughout your application:\n\n .. code-block:: typescript\n :caption: src/db/index.ts\n\n import { drizzle } from 'drizzle-orm/gel';\n import { createClient } from 'gel-js';\n import * as schema from '@/drizzle/schema';\n import * as relations from '@/drizzle/relations';\n\n // Import our schema\n import * as schema from './schema';\n\n // Initialize Gel client\n const gelClient = createClient();\n\n // Create Drizzle instance\n export const db = drizzle({\n client: gelClient,\n schema: {\n ...schema,\n ...relations\n },\n });\n\n // Helper types for use in our application\n export type Book = typeof schema.book.$inferSelect;\n export type NewBook = typeof schema.book.$inferInsert;\n\n export type Note = typeof schema.note.$inferSelect;\n export type NewNote = typeof schema.note.$inferInsert;\n\n\nPerform database operations with Drizzle\n========================================\n\nFor more detailed information on querying and other operations, refer to the `Drizzle documentation `_. Below are some examples of common database operations you can perform with Drizzle.\n\n.. edb:split-section::\n\n Drizzle provides a clean, type-safe API for database operations. Here are some examples of common operations:\n\n **Selecting data:**\n\n .. code-block:: typescript\n\n // Get all books with their notes\n const allBooks = await db.query.book.findMany({\n with: {\n notes: true,\n },\n });\n\n // Get a specific book\n const book = await db.query.book.findFirst({\n where: eq(books.id, id),\n with: { notes: true },\n });\n\n\n.. edb:split-section::\n\n **Inserting data:**\n\n .. code-block:: typescript\n\n // Insert a new book\n const newBook = await db.insert(book).values({\n title: 'The Great Gatsby',\n author: 'F. Scott Fitzgerald',\n year: 1925,\n genre: 'Novel',\n }).returning();\n\n // Insert a note for a book\n const newNote = await db.insert(note).values({\n text: 'A classic novel about the American Dream',\n book_id: newBook.bookId,\n }).returning();\n\n **Bulk inserting data:**\n\n .. code-block:: typescript\n\n // Insert multiple books at once\n const newBooks = await db.insert(book).values([\n {\n title: '1984',\n author: 'George Orwell',\n year: 1949,\n genre: 'Dystopian',\n },\n {\n title: 'To Kill a Mockingbird',\n author: 'Harper Lee',\n year: 1960,\n genre: 'Fiction',\n },\n {\n title: 'Pride and Prejudice',\n author: 'Jane Austen',\n year: 1813,\n genre: 'Romance',\n },\n ]).returning();\n\n.. edb:split-section::\n\n **Updating data:**\n\n .. code-block:: typescript\n\n // Update a book\n const updatedBook = await db.update(book)\n .set({\n title: 'Updated Title',\n author: 'Updated Author',\n })\n .where(eq(books.id, bookId))\n .returning();\n\n\n.. edb:split-section::\n\n **Deleting data:**\n\n .. code-block:: typescript\n\n // Delete a note\n await db.delete(notes).where(eq(notes.id, noteId));\n\n\nUsing Drizzle with Next.js\n==========================\n\n.. edb:split-section::\n\n In a Next.js application, you can use your Drizzle client in API routes and server components. Here's an example of an API route that gets all books:\n\n .. code-block:: typescript\n :caption: src/app/api/books/route.ts\n\n import { NextResponse } from 'next/server';\n import { db } from '@/db';\n\n export async function GET() {\n try {\n const allBooks = await db.query.book.findMany({\n with: { notes: true },\n });\n\n return NextResponse.json(allBooks);\n } catch (error) {\n console.error('Error fetching books:', error);\n return NextResponse.json(\n { error: 'Failed to fetch books' },\n { status: 500 }\n );\n }\n }\n\n\n.. edb:split-section::\n\n And here's an example of using Drizzle in a server component:\n\n .. code-block:: typescript\n :caption: src/app/books/page.tsx\n\n import { db } from '@/db';\n import BookCard from '@/components/BookCard';\n\n export default async function BooksPage() {\n const books = await db.query.book.findMany({\n with: { notes: true },\n });\n\n return (\n
\n {books.map((book) => (\n \n ))}\n
\n );\n }\n\n\nKeep going!\n===========\n\nYou are now ready to use Gel with Drizzle in your applications. This integration gives you the best of both worlds: Gel's powerful features and Drizzle's type-safe, intuitive API.\n\nFor a complete example of using Gel with Drizzle in a Next.js application, check out our `Book Notes app example `_.\n\nYou can also find a detailed tutorial on building a Book Notes app with Gel, Drizzle, and Next.js in our :ref:`documentation `.\n" }, { "path": "docs/intro/guides/index.rst", "content": "======\nGuides\n======\n\n.. toctree::\n :maxdepth: 1\n\n ai/index\n drizzle/index\n" }, { "path": "docs/intro/index.rst", "content": ".. _ref_intro:\n\n==============\nWelcome to Gel\n==============\n\n.. toctree::\n :maxdepth: 3\n :hidden:\n\n installation\n quickstart/index\n tutorials/index\n guides/index\n localdev\n cli\n instances\n projects\n schema\n migrations\n branches\n edgeql\n clients\n\n|Gel| is a next-generation `graph-relational database\n`_ designed\nas a spiritual successor to the relational database.\n\nIt inherits the strengths of SQL databases: type safety, performance,\nreliability, and transactionality. But instead of modeling data in a\nrelational (tabular) way, Gel represents data with *object types*\ncontaining *properties* and *links* to other objects. It leverages\nthis object-oriented model to provide a superpowered query language that\nsolves some of SQL's biggest usability problems.\n\nHow to read the docs\n^^^^^^^^^^^^^^^^^^^^\n\n|Gel| is a complex system, but we've structured the documentation so you can\nlearn it in \"phases\". You only need to learn as much as you need to start\nbuilding your application.\n\n- **Get Started** ā€”\n Start with the :ref:`quickstart `. It walks\n through Gel's core workflows: how to install Gel, create an instance,\n write a simple schema, execute a migration, write some simple queries, and\n use the client libraries. The rest of the section goes deeper on each of\n these subjects.\n\n- **Schema** ā€”\n A set of pages that break down the concepts of syntax of Gel's schema\n definition language (SDL). This starts with a rundown of Gel's primitive\n type system (:ref:`Primitives `), followed by a\n description of (:ref:`Object Types `) and the\n things they can contain: links, properties, indexes, access policies, and\n more.\n\n- **EdgeQL** ā€”\n A set of pages that break down Gel's query language, EdgeQL. It starts\n with a rundown of how to declare :ref:`literal values `,\n then introduces some key EdgeQL concepts like sets, paths, and type casts.\n With the basics established, it proceeds to break down all of EdgeQL's\n top-level statements: ``select``, ``insert``, and so on.\n\n- **Guides** ā€”\n Contains collections of guides on topics that are peripheral to Gel\n itself: how to deploy to various cloud providers, how to integrate with\n various frameworks, and how to introspect the schema to build\n code-generation tools on top of Gel.\n\n- **Standard Library** ā€”\n This section contains an encyclopedic breakdown of Gel's built-in types\n and the functions/operators that can be used with them. We didn't want to \\\n clutter the **EdgeQL** section with all the nitty-gritty on each of these.\n If you're looking for a particular function (say, a ``replace``), go to the\n Standard Library page for the relevant type (in this case, :ref:`String\n `), and peruse the table for what you're looking for\n (:eql:func:`str_replace`).\n\n- **Client Libraries**\n The documentation for Gel's set of official client libraries for\n JavaScript/TypeScript, Python, Go, and Rust. All client libraries implement\n Gel's binary protocol and provide a standard interface for executing\n queries. If you're using another language, you can execute queries\n :ref:`over HTTP `. This section also includes\n documentation for Gel's :ref:`GraphQL ` endpoint.\n\n- **CLI**\n Complete reference for the |gelcmd| command-line tool. The CLI is\n self-documentingā€”add the ``--help`` flag after any command to print the\n relevant documentationā€”so you shouldn't need to reference this section often.\n\n- **Reference**\n The *Reference* section contains a complete breakdown of Gel's *syntax*\n (for both EdgeQL and SDL), *internals* (like the binary protocol and dump\n file format), and *configuration settings*. Usually you'll only need to\n reference these once you're an advanced user.\n\n- **Changelog**\n Detailed changelogs for each successive version of Gel, including any\n breaking changes, new features, bigfixes, and links to\n\n\nTooling\n^^^^^^^\n\nTo actually build apps with Gel, you'll need to know more than SDL and\nEdgeQL.\n\n- **CLI** ā€”\n The most commonly used CLI functionality is covered in the :ref:`Quickstart\n `. For additional details, we have dedicated guides for\n :ref:`Migrations ` and :ref:`Projects\n `. A full CLI reference is available under\n :ref:`CLI `.\n\n- **Client Libraries** ā€”\n To actually execute queries, you'll use one of our client libraries for\n JavaScript, Go, or Python; find your preferred library under :ref:`Client\n Libraries `. If you're using another language, you can\n still use Gel! You can execute :ref:`queries via HTTP `.\n- **Deployment** ā€”\n To publish a Gel-backed application, you'll need to deploy Gel. Refer\n to :ref:`Guides > Deployment ` for step-by-step\n deployment guides for all major cloud hosting platforms, as well as\n instructions for self-hosting with Docker.\n\n.. .. eql:react-element:: DocsNavTable\n\n|Gel| features:\n\n.. class:: ticklist\n\n- strict, strongly typed schema;\n- powerful and clean query language;\n- ability to easily work with complex hierarchical data;\n- built-in support for schema migrations.\n\n|Gel| is not a graph database: the data is stored and queried using\nrelational database techniques. Unlike most graph databases, Gel\nmaintains a strict schema.\n\n|Gel| is not a document database, but inserting and querying hierarchical\ndocument-like data is trivial.\n\n|Gel| is not a traditional object database, despite the classification,\nit is not an implementation of OOP persistence.\n\n" }, { "path": "docs/intro/install_table.rst", "content": ".. tabs::\n\n .. code-tab:: bash\n :caption: bash\n\n $ curl https://www.geldata.com/sh --proto \"=https\" -sSf1 | sh\n\n .. code-tab:: powershell\n :caption: Powershell\n\n PS> irm https://www.geldata.com/ps1 | iex\n\n .. code-tab:: bash\n :caption: Homebrew\n\n $ brew install geldata/tap/gel-cli\n\n .. code-tab:: bash\n :caption: Nixpkgs\n\n $ nix-shell -p gel\n\n .. code-tab:: bash\n :caption: JavaScript\n\n $ npx gel --version\n\n .. code-tab:: bash\n :caption: Python\n\n $ uvx gel --version\n\n" }, { "path": "docs/intro/installation.rst", "content": ".. _ref_cli_gel_install:\n\n============\nInstallation\n============\n\nWe provide a :ref:`CLI for managing and interacting with local and remote databases `. If you're using JavaScript or Python, our client libraries will handle downloading and running the CLI for you using tools like ``npx`` and ``uvx``.\n\nFor everyone else, or if you wish to install the CLI globally, you can install using our bash installer or your operating system's package manager.\n\n.. include:: ./install_table.rst" }, { "path": "docs/intro/instances.rst", "content": ".. _ref_intro_instances:\n\n=========\nInstances\n=========\n\nLet's get to the good stuff. You can spin up a Gel instance with a single\ncommand.\n\n.. code-block:: bash\n\n $ gel instance create my_instance\n\nThis creates a new instance named ``my_instance`` that runs the latest stable\nversion of Gel. (Gel itself will be automatically installed if it isn't\nalready.) Alternatively you can specify a specific version with\n``--version``.\n\n.. code-block:: bash\n\n $ gel instance create my_instance --version 6.1\n $ gel instance create my_instance --version nightly\n\nWe can execute a query against our new instance with :gelcmd:`query`. Specify\nwhich instance to connect to by passing an instance name into the ``-I`` flag.\n\n.. code-block:: bash\n\n $ gel query \"select 3.14\" -I my_instance\n 3.14\n\nManaging instances\n^^^^^^^^^^^^^^^^^^\nInstances can be stopped, started, restarted, and destroyed.\n\n.. code-block:: bash\n\n $ gel instance stop -I my_instance\n $ gel instance start -I my_instance\n $ gel instance restart -I my_instance\n $ gel instance destroy -I my_instance\n\n\nListing instances\n^^^^^^^^^^^^^^^^^\n\nTo list all instances on your machine:\n\n.. code-block:: bash\n\n $ gel instance list\n ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”\n ā”‚ Kind ā”‚ Name ā”‚ Port ā”‚ Version ā”‚ Status ā”‚\n ā”œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¼ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¼ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¼ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¼ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¤\n ā”‚ local ā”‚ my_instance ā”‚ 10700 ā”‚ x.x+cc4f3b5 ā”‚ active ā”‚\n ā”‚ local ā”‚ my_instance_2 ā”‚ 10701 ā”‚ x.x+cc4f3b5 ā”‚ active ā”‚\n ā”‚ local ā”‚ my_instance_3 ā”‚ 10702 ā”‚ x.x+cc4f3b5 ā”‚ active ā”‚\n ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜\n\nFurther reference\n^^^^^^^^^^^^^^^^^\n\nFor complete documentation on managing instances with the CLI (upgrading,\nviewing logs, etc.), refer to the :ref:`gel instance\n` reference or view the help text in your shell:\n\n.. code-block:: bash\n\n $ gel instance --help\n" }, { "path": "docs/intro/localdev.rst", "content": "=================\nLocal Development\n=================\n\nOne of Gel's most powerful features is its seamless support for local development. The Gel CLI makes it incredibly easy to spin up a local instance, manage it, access GUI, and iterate on your schema quickly and safely. This guide outlines the flexible options available for your local development workflow.\n\nIf you're using JavaScript or Python, our client libraries will automatically handle the installation for you using tools like ``npx`` and ``uvx``. For other environments or to install the CLI globally, you can use one of the following methods:\n\n.. include:: ./install_table.rst\n\n\nInitialize your local instance\n==============================\n\nIt's easy to get started with a local Gel instance. Navigate to the root of your project repository and run:\n\n.. code-block:: bash\n\n $ gel init\n\nCreates a database tied to the current directory and to the :ref:`gel.toml ` file in it. This simplifies connection configuration and installation for you. Alias for :ref:`gel project init `.\n\nTo conserve resources, Gel automatically puts inactive local development instances to sleep. This means you can have multiple instances running without them draining your system's resources when not in use.\n\nIterate on your schema\n======================\n\nGel simplifies the process of evolving your data model. You can apply changes from your Gel schema files directly to your running local instance without needing to create a separate migration file for every minor adjustment.\n\nThere are two primary ways to apply schema changes during development:\n\n1. **Automatic updates with** :gelcmd:`watch --migrate`\n For a hands-off approach, you can use the watch command. This starts a process that monitors your Gel schema files for changes and automatically migrates your local instance as soon as you save them:\n\n .. code-block:: bash\n\n $ gel watch --migrate\n\n This is ideal for rapid iteration when you want to see your schema changes reflected immediately.\n\n2. **Manual updates with** :gelcmd:`migrate --dev-mode`\n If you prefer more explicit control, or don't want a background process running, you can apply schema changes manually:\n\n .. code-block:: bash\n\n $ gel migrate --dev-mode\n\n This command performs the same action as the watch --migrate modeā€”applying the current state of your schema files to the local instanceā€”but only when you explicitly run it.\n\nFinalizing changes\n==================\n\nOnce you're satisfied with the schema changes you've made iteratively, you'll want to create a migration file which will be committed to version control and shared with others. This new migration file will encapsulate all the modifications made since your last migration.\n\n1. **Create the migration file**\n .. code-block:: bash\n\n $ gel migration create\n\n This command inspects the differences between your last migration file and the current state of your database schema, then generates a new migration file reflecting these changes.\n\n1. **Align your local instance**\n After creating the migration, run the following command to ensure your local instance's migration history is aligned with this new migration. You can do this by running:\n\n .. code-block:: bash\n\n $ gel migrate --dev-mode\n\n This command effectively \"fast-forwards\" your local instance. From its perspective, it will appear as though all the iterative changes were applied as part of this single, new migration. This keeps your local development environment consistent with the migration history you'll use in other environments (like staging or production).\n\nUndoing destructive changes\n===========================\n\nMistakes happen! You might accidentally make a destructive schema change. Fortunately, Gel has your back. Every time you migrate your schema (either via :gelcmd:`watch --migrate` or :gelcmd:`migrate --dev-mode`), a backup of your local instance is automatically taken.\nThis behavior can be disabled by setting env variable ``GEL_AUTO_BACKUP_MODE`` to ``disabled``.\n\nIf you need to roll back to a previous state:\n\n1. **Stop any active migration processes**: Ensure :gelcmd:`watch --migrate` is not running.\n\n2. **Find the backup ID**: Look through your shell's scrollback history. You'll find messages indicating backups were made, along with their IDs. Identify the ID of the backup created before the destructive change. You can also use the :gelcmd:`instance listbackups` command to list all backups for this instance.\n\n3. **Restore the instance**\n .. code-block:: bash\n\n $ gel instance restore -I \n\n Replace with the actual ID and with the name of your instance (e.g., my_project). This will restore both your data and schema to the state at that backup point.\n\nOnce restored, you can make the intended schema changes and then restart :gelcmd:`watch --migrate` or use :gelcmd:`migrate --dev-mode` as preferred.\n\nKeeping code in sync\n====================\n\nMany Gel language bindings offer code generation capabilities (e.g., query builders, typed query functions). This generated code needs to stay synchronized with your schema. Gel provides a system of hooks and watchers that you can configure in your |gel.toml| file to automate this.\n\nThese hooks can trigger codegen scripts when:\n\n* The schema changes (using the \"schema.change.after\" hook).\n* Specific files are edited (using watch scripts).\n\nHere's an example |gel.toml| configuration for a TypeScript project. It runs a query builder generator and a queries generator at the appropriate times:\n\n.. code-block:: toml\n\n [instance]\n server-version = \"6.7\"\n\n [hooks]\n \"schema.change.after\" = \"npx @gel/generate edgeql-js && npx @gel/generate queries\"\n\n [watch]\n \"src/queries/**/*.edgeql\" = \"npx @gel/generate queries\"\n\nExplanation:\n\n* ``[hooks] / \"schema.change.after\"``: When any schema change is successfully applied, we run the query builder generator (to reflect schema structure changes) and the queries generator (to update based on new or modified types).\n* ``[watch] / \"src/queries/**/*.edgeql\"``: If any ``.edgeql`` files within the ``src/queries/`` directory (or its subdirectories) are modified, the command ``npx @gel/generate queries`` is executed. This ensures that your typed query functions are always up-to-date with your EdgeQL query definitions.\n\nBy configuring these hooks and watchers, you can maintain a smooth workflow where your generated code automatically adapts to changes in your schema and query files.\n" }, { "path": "docs/intro/migrations.rst", "content": ".. _ref_intro_migrations:\n\n==========\nMigrations\n==========\n\n.. index:: fill_expr, cast_expr\n\n|Gel's| baked-in migration system lets you painlessly evolve your schema\nthroughout the development process. If you want to work along with this guide,\nstart a new project with :ref:`ref_cli_gel_project_init`. This will create a\nnew instance and some empty schema files to get you started.\n\n\n1. Start the ``watch`` command\n------------------------------\n\nThe easiest way to work with your schema in development is by running\n:gelcmd:`watch --migrate`. This long-running task will monitor your schema files and\nautomatically apply schema changes in your database as you work.\n\n.. code-block:: bash\n\n $ gel watch --migrate\n Hint: --migrate will apply any changes from your schema files to the database.\n When ready to commit your changes, use:\n 1) `gel migration create` to write those changes to a migration file,\n 2) `gel migrate --dev-mode` to replace all synced changes with the migration.\n\n Monitoring /home/instancename for changes in:\n\n --migrate: gel migration apply --dev-mode\n\n\nIf you get output similar to the output above, you're ready to get started!\n\n\n2. Write an initial schema\n--------------------------\n\nBy convention, your Gel schema is defined inside one or more |.gel|\nfiles that live in a directory called ``dbschema`` in the root directory of\nyour codebase.\n\n.. code-block::\n\n .\n ā”œā”€ā”€ dbschema\n ā”‚ ā””ā”€ā”€ default.gel # schema file (written by you)\n ā””ā”€ā”€ gel.toml\n\nThe schema itself is written using Gel's schema definition language. Edit\nyour :dotgel:`dbschema/default` and add the following schema inside your\n``module default`` block:\n\n.. code-block:: sdl\n\n type User {\n required name: str;\n }\n\n type Post {\n required title: str;\n required author: User;\n }\n\n\nIt's common to keep your entire schema in a single file, and many users use\nthis :dotgel:`default` that is created when you start a project. However it's\nalso possible to split their schemas across a number of |.gel| files.\n\nOnce you save your initial schema, assuming it is valid, the ``watch`` command\nwill pick it up and apply it to your database.\n\n\n3. Edit your schema files\n-------------------------\n\nAs your application evolves, directly edit your schema files to reflect your\ndesired data model. Try updating your :dotgel:`dbschema/default` to add a\n``Comment`` type:\n\n.. code-block:: sdl-diff\n\n type User {\n required name: str;\n }\n\n type Post {\n required title: str;\n required author: User;\n }\n\n + type Comment {\n + required content: str;\n + }\n\nWhen you save your changes, ``watch`` will immediately begin applying your\nnew schema to the database.\n\n.. note::\n\n If your schema cannot be applied, the ``watch`` command will generate an\n error. If you're using one of our client bindings as you update your schema\n with ``watch``, you will see the error there the next time you execute a\n query using that client binding.\n\n If things aren't working the way you expect after making a schema change,\n take a look at the ``watch`` console to find out why.\n\nOnce you have the schema the way you want it, and you're ready to lock it in\nand commit it to version control, it's time to generate a migration.\n\n\n4. Generate a migration\n-----------------------\n\nTo generate a migration that reflects all your changes, run :gelcmd:`migration\ncreate`.\n\n.. code-block:: bash\n\n $ gel migration create\n\n\nThe CLI reads your schema file and sends it to the active Gel instance. The\ninstance compares the file's contents to its current schema state and\ndetermines a migration plan. **The migration plan is generated by the\ndatabase itself.**\n\nThis plan is then presented to you interactively; each detected schema change\nwill be individually presented to you for approval. For each prompt, you have\na variety of commands at your disposal. Type ``y`` to approve, ``n`` to\nreject, ``q`` to cancel the migration, or ``?`` for a breakdown of some more\nadvanced options.\n\n.. code-block:: bash\n\n $ gel migration create\n did you create object type 'default::Comment'? [y,n,l,c,b,s,q,?]\n > y\n did you create object type 'default::User'? [y,n,l,c,b,s,q,?]\n > y\n did you create object type 'default::Post'? [y,n,l,c,b,s,q,?]\n > y\n Created dbschema/migrations/00001.edgeql, id: \n\n\n.. _ref_intro_migrations_wo_iteration:\n\nMigration without iteration\n---------------------------\n\nIf you want to change the schema, but you already know exactly what you want to\nchange and don't need to iterate on your schema ā€” you want to lock in the\nmigration right away ā€” :gelcmd:`watch` might not be the tool you reach for.\n\nInstead, you might use this method:\n\n1. Edit your schema files\n2. Create your migration with :gelcmd:`migration create`\n3. Apply your migration with :gelcmd:`migrate`\n\nSince you're not using ``watch``, the schema changes are not applied when you\nsave your schema files. As a result, we need to tack an extra step on the end\nof the process of applying the migration. That's handled by :gelcmd:`migrate`.\n\n.. code-block:: bash\n\n $ gel migrate\n Applied m1virjowa... (00002.edgeql)\n\nOnce your migration is applied, you'll see the schema changes reflected in your\ndatabase.\n\n\nData migrations\n---------------\n\nDepending on how the schema was changed, data in your database may prevent\n|Gel| from applying your schema changes. Imagine we added a required ``body``\nproperty to our ``Post`` type:\n\n.. code-block:: sdl-diff\n\n type User {\n required name: str;\n }\n\n type Post {\n required title: str;\n + required body: str;\n required author: User;\n }\n\n type Comment {\n required content: str;\n }\n\nIf we hadn't added any ``Post`` objects to our database before this, everything\nwould have worked fine, but it's likely that, in testing out our schema, we\n*did* add a ``Post`` object. It does not have a ``body`` property, but now\nwe've told the database this property is required on all ``Post`` objects. The\ndatabase can't apply this change because existing data would break it.\n\nWe have a couple of options here. We could delete all the offending objects.\n\n.. code-block:: edgeql-repl\n\n db> delete Post;\n {\n default::Post {id: a4a0a40c-d9f5-11ed-8912-1397f7af9fdf},\n default::Post {id: cc051bea-d9f5-11ed-a26d-2b64b6b273a4}\n }\n\nNow, if we save the schema again, :gelcmd:`watch` will be able to apply it. If\nwe have data in here we don't want to lose though, that's not a good option. In\nthat case, we might drop back to creating and applying the migration outside of\n:gelcmd:`watch`.\n\nTo start, run :gelcmd:`migration create`. The interactive plan generator will\nask you for an EdgeQL expression to map the contents of your database to the\nnew schema.\n\n.. code-block:: bash\n\n $ gel migration create\n did you create property 'body' of object type\n 'default::Post'? [y,n,l,c,b,s,q,?]\n > y\n Please specify an expression to populate existing objects in order to make\n property 'body' of object type 'default::Post' required:\n fill_expr>\n\nBecause the ``body`` property does not currently exist, the database contains\n``Post`` objects without it. The expression you provide will be used to *assign\na body* to any ``Post`` object that doesn't have one. We'll just provide a\nsimple default: ``'No content'``.\n\n.. code-block::\n\n fill_expr> 'No content'\n Created dbschema/migrations/00002.edgeql, id:\n m1pjiibv4sa4cao7txpgsbuw2erctmacyrj4qmn45ggapsaztmvxfa\n\nNice! It accepted our answer and created a new migration file\n``00002.edgeql``. Let's see what the newly created ``00002.edgeql`` file\ncontains.\n\n.. code-block:: edgeql\n\n CREATE MIGRATION m1pjiibv4sa4cao7txpgsbuw2erctmacyrj4qmn45ggapsaztmvxfa\n ONTO m1nlvzbm7buwktkp4vu4shylq6zp2shruokbbssyeidqmmmfqz77yq\n {\n ALTER TYPE default::Post {\n CREATE REQUIRED PROPERTY body: std::str {\n SET REQUIRED USING ('No content');\n };\n };\n };\n\nWe have a ``CREATE MIGRATION`` block containing an ``ALTER TYPE`` statement to\ncreate ``Post.body`` as a ``required`` property. We can see that our fill\nexpression (``'No content'``) is included directly in the migration file.\n\nNote that we could have provide an *arbitrary EdgeQL expression*! The\nfollowing EdgeQL features are often useful:\n\n.. list-table::\n\n * - ``assert_exists``\n - This is an \"escape hatch\" function that tells Gel to assume the input\n has *at least* one element.\n\n .. code-block::\n\n fill_expr> assert_exists(.body)\n\n If you provide a ``fill_expr`` like the one above, you must separately\n ensure that all posts have a ``body`` before executing the migration;\n otherwise it will fail.\n\n * - ``assert_single``\n - This tells Gel to assume the input has *at most* one element. This\n will throw an error if the argument is a set containing more than one\n element. This is useful is you are changing a property from ``multi`` to\n ``single``.\n\n .. code-block::\n\n fill_expr> assert_single(.sheep)\n\n * - type casts\n - Useful when converting a property to a different type.\n\n .. code-block::\n\n cast_expr> .xp\n\n\nFurther reading\n^^^^^^^^^^^^^^^\n\n- :ref:`Guide to schema migrations `\n- :ref:`Migration tips `\n\nFurther information can be found in the :ref:`CLI\nreference ` or the `Beta 1 blog post\n`_,\nwhich describes the design of the migration system.\n" }, { "path": "docs/intro/projects.rst", "content": ".. _ref_intro_projects:\n\n========\nProjects\n========\n\nIt can be inconvenient to pass the ``-I`` flag every time you wish to run a\nCLI command.\n\n.. code-block:: bash\n\n $ gel migration create -I my_instance\n\nThat's one of the reasons we introduced the concept of an *Gel\nproject*. A project is a directory on your file system that is associated\n(\"linked\") with a Gel instance.\n\n.. note::\n\n Projects are intended to make *local development* easier! They only exist on\n your local machine and are managed with the CLI. When deploying Gel for\n production, you will typically pass connection information to the client\n library using environment variables.\n\nWhen you're inside a project, all CLI commands will be applied against the\n*linked instance* by default (no CLI flags required).\n\n.. code-block:: bash\n\n $ gel migration create\n\nThe same is true for all Gel client libraries (discussed in more depth in\nthe :ref:`Clients ` section). If the following file lives\ninside a Gel project directory, ``createClient`` will discover the project\nand connect to its linked instance with no additional configuration.\n\n.. code-block:: typescript\n\n // clientTest.js\n import {createClient} from 'gel';\n\n const client = createClient();\n await client.query(\"select 5\");\n\nInitializing\n^^^^^^^^^^^^\n\nTo initialize a project, create a new directory and run :gelcmd:`project init`\ninside it. You'll see something like this:\n\n.. code-block:: bash\n\n $ gel project init\n No `gel.toml` found in this repo or above.\n Do you want to initialize a new project? [Y/n]\n > Y\n Specify the name of Gel instance to use with this project\n [default: my_instance]:\n > my_instance\n Checking Gel versions...\n Specify the version of Gel to use with this project [default: x.x]:\n > # (left blank for default)\n ...\n Successfully installed x.x+cc4f3b5\n Initializing Gel instance...\n Applying migrations...\n Everything is up to date. Revision initial\n Project initialized.\n To connect to my_instance, run `gel`\n\nThis command does a couple important things.\n\n1. It spins up a new Gel instance called ``my_instance``.\n2. If no |gel.toml| file exists, it will create one. This is a\n configuration file that marks a given directory as a Gel project. Learn\n more about it in the :ref:`gel.toml reference `.\n\n .. code-block:: toml\n\n [instance]\n server-version = \"6.0\"\n\n3. If no ``dbschema`` directory exists, it will be created, along with an\n empty :dotgel:`default` file which will contain your schema. If a\n ``dbschema`` directory exists and contains a subdirectory called\n ``migrations``, those migrations will be applied against the new instance.\n\nEvery project maps one-to-one to a particular Gel instance. From\ninside a project directory, you can run :gelcmd:`project info` to see\ninformation about the current project.\n\n.. code-block:: bash\n\n $ gel project info\n ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”\n ā”‚ Instance name ā”‚ my_instance ā”‚\n ā”‚ Project root ā”‚ /path/to/project ā”‚\n ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜\n\n\nConnection\n^^^^^^^^^^\n\nAs long as you are inside the project directory, all CLI commands will be\nexecuted against the project-linked instance. For instance, you can simply run\n|gelcmd| to open a REPL.\n\n.. code-block:: bash\n\n $ gel\n Gel x.x+cc4f3b5 (repl x.x+da2788e)\n Type \\help for help, \\quit to quit.\n my_instance:main> select \"Hello world!\";\n\nBy contrast, if you leave the project directory, the CLI will no longer know\nwhich instance to connect to. You can solve this by specifing an instance name\nwith the ``-I`` flag.\n\n.. code-block:: bash\n\n $ cd ~\n $ gel\n gel error: no `gel.toml` found and no connection options are specified\n Hint: Run `gel project init` or use any of `-H`, `-P`, `-I` arguments to\n specify connection parameters. See `--help` for details\n $ gel -I my_instance\n Gel x.x+cc4f3b5 (repl x.x+da2788e)\n Type \\help for help, \\quit to quit.\n my_instance:main>\n\nSimilarly, client libraries will auto-connect to the project's\nlinked instance without additional configuration.\n\nUsing remote instances\n^^^^^^^^^^^^^^^^^^^^^^\n\nYou may want to initialize a project that points to a remote Gel instance.\nThis is totally a valid case and Gel fully supports it! Before running\n:gelcmd:`project init`, you just need to create an alias for the remote\ninstance using :gelcmd:`instance link`, like so:\n\n.. lint-off\n\n.. code-block:: bash\n\n $ gel instance link\n Specify server host [default: localhost]:\n > 192.168.4.2\n Specify server port [default: 5656]:\n > 10818\n Specify database user [default: admin]:\n > admin\n Specify branch [default: main]:\n > main\n Unknown server certificate: SHA1:c38a7a90429b033dfaf7a81e08112a9d58d97286.\n Trust? [y/N]\n > y\n Password for 'admin':\n Specify a new instance name for the remote server [default: abcd]:\n > staging_db\n Successfully linked to remote instance. To connect run:\n gel -I staging_db\n\n.. lint-on\n\nAfter receiving the necessary connection information, this command links the\nremote instance to a local alias ``\"staging_db\"``. You can use this as\ninstance name in CLI commands.\n\n.. code-block::\n\n $ gel -I staging_db\n gel>\n\nTo initialize a project that uses the remote instance, provide this alias when\nprompted for an instance name during the :gelcmd:`project init` workflow.\n\n\nUnlinking\n^^^^^^^^^\n\nAn instance can be unlinked from a project. This leaves the instance running\nbut effectively \"uninitializes\" the project. The |gel.toml| and\n``dbschema`` are left untouched.\n\n.. code-block:: bash\n\n $ gel project unlink\n\nIf you wish to delete the instance as well, use the ``-D`` flag.\n\n.. code-block:: bash\n\n $ gel project unlink -D\n\nUpgrading\n^^^^^^^^^\n\nA standalone instance (not linked to a project) can be upgraded with the\n:gelcmd:`instance upgrade` command.\n\n.. code-block:: bash\n\n $ gel project upgrade --to-latest\n $ gel project upgrade --to-nightly\n $ gel project upgrade --to-version x.x\n\n\nSee info\n^^^^^^^^\n\nYou can see the location of a project and the name of its linked instance.\n\n.. code-block:: bash\n\n $ gel project info\n ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”\n ā”‚ Instance name ā”‚ my_app ā”‚\n ā”‚ Project root ā”‚ /path/to/my_app ā”‚\n ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜\n" }, { "path": "docs/intro/quickstart/ai/fastapi.rst", "content": ".. _ref_quickstart_ai:\n\n======================\nUsing the built-in RAG\n======================\n\n.. edb:split-section::\n\n In this section we'll learn about |Gel's| built-in vector search and\n retrieval-augmented generation capabilities. We'll be continuing from where\n we left off in the :ref:`main quickstart `. Feel free to browse the\n complete flascards app code in this `repo\n `_.\n\n In this tutorial we'll focus on creating a ``/fetch_similar`` endpoint for\n looking up flashcards similar to a text search query, as well as a\n ``/fetch_rag`` endpoint that's going to enable us to talk to an LLM about\n the content of our flashcard deck.\n\n We're going to start with the same schema we left off with in the primary\n quickstart.\n\n\n .. code-block:: sdl\n :caption: dbschema/default.gel\n\n module default {\n abstract type Timestamped {\n required created_at: datetime {\n default := datetime_of_statement();\n };\n required updated_at: datetime {\n default := datetime_of_statement();\n };\n }\n\n type Deck extending Timestamped {\n required name: str;\n description: str;\n\n multi cards: Card {\n constraint exclusive;\n on target delete allow;\n };\n };\n\n type Card extending Timestamped {\n required order: int64;\n required front: str;\n required back: str;\n }\n }\n\n\n.. edb:split-section::\n\n AI-related features in |Gel| come packaged in the extension called ``ai``.\n Let's enable it by adding the following line on top of the\n :dotgel:`dbschema/default` and running a migration.\n\n This does a few things. First, it enables us to use features from the extension by prefixing them with ``ext::ai::``.\n\n\n .. code-block:: sdl-diff\n :caption: dbschema/default.gel\n\n + using extension ai;\n\n module default {\n abstract type Timestamped {\n required created_at: datetime {\n default := datetime_of_statement();\n };\n required updated_at: datetime {\n default := datetime_of_statement();\n };\n }\n\n type Deck extending Timestamped {\n required name: str;\n description: str;\n\n multi cards: Card {\n constraint exclusive;\n on target delete allow;\n };\n };\n\n type Card extending Timestamped {\n required order: int64;\n required front: str;\n required back: str;\n }\n }\n\n.. edb:split-section::\n\n This enabled us to use features in the ``ext::ai::`` namespace. Here's a\n notable one: ``ProviderConfig``, which we can use to configure our API\n keys. |Gel| supports a variety of external APIs for creating embedding\n vectors for text and fetching LLM completions.\n\n Let's configure an API key for OpenAI by running the following query in the\n REPL:\n\n .. note::\n\n Once the extension is active, we can also access the dedicated AI tab\n in the UI. There we can manage provider configurations and try out\n different RAG configuraton in the Playground.\n\n\n .. code-block:: edgeql-repl\n\n db> configure current database\n insert ext::ai::OpenAIProviderConfig {\n secret := 'sk-....',\n };\n\n\n.. edb:split-section::\n\n Once last thing before we move on. Let's add some sample data to give the\n embedding model something to work with. You can copy and run this command\n in the terminal, or come up with your own sample data.\n\n\n .. code-block:: edgeql\n :class: collapsible\n\n $ cat << 'EOF' | gel query --file -\n with deck := (\n insert Deck {\n name := 'Smelly Cheeses',\n description := 'To impress everyone with stinky cheese trivia.'\n }\n )\n for card_data in {(\n 1,\n 'Ɖpoisses de Bourgogne',\n 'Known as the \"king of cheeses\", this French cheese is so pungent it\\'s banned on public transport in France. Washed in brandy, it becomes increasingly funky as it ages. Orange-red rind, creamy interior.'\n ), (\n 2,\n 'Vieux-Boulogne',\n 'Officially the smelliest cheese in the world according to scientific studies. This northern French cheese has a reddish-orange rind from being washed in beer. Smooth, creamy texture with a powerful aroma.'\n ), (\n 3,\n 'Durian Cheese',\n 'This Malaysian creation combines durian fruit with cheese, creating what some consider the ultimate \"challenging\" dairy product. Combines the pungency of blue cheese with durian\\'s notorious aroma.'\n ), (\n 4,\n 'Limburger',\n 'German cheese famous for its intense smell, often compared to foot odor due to the same bacteria. Despite its reputation, has a surprisingly mild taste with notes of mushroom and grass.'\n ), (\n 5,\n 'Roquefort',\n 'The \"king of blue cheeses\", aged in limestone caves in southern France. Contains Penicillium roqueforti mold. Strong, tangy, and salty with a crumbly texture. Legend says it was discovered when a shepherd left his lunch in a cave.'\n ), (\n 6,\n 'What makes washed-rind cheeses so smelly?',\n 'The process of washing cheese rinds in brine, alcohol, or other solutions promotes the growth of Brevibacterium linens, the same bacteria responsible for human body odor. This bacteria contributes to both the orange color and distinctive aroma.'\n ), (\n 7,\n 'Stinking Bishop',\n 'Named after the Stinking Bishop pear (not a religious figure). This English cheese is washed in perry made from these pears. Known for its powerful aroma and sticky, pink-orange rind. Gained fame after being featured in Wallace & Gromit.'\n )}\n union (\n insert Card {\n deck := deck,\n order := card_data.0,\n front := card_data.1,\n back := card_data.2\n }\n );\n EOF\n\n\n.. edb:split-section::\n\n Now we can finally start producing embedding vectors. Since |Gel| is fully\n aware of when your data gets inserted, updated and deleted, it's perfectly\n equipped to handle all the tedious work of keeping those vectors up to\n date. All that's left for us is to create a special ``deferred index`` on\n the data we would like to perform similarity search on.\n\n\n .. code-block:: sdl-diff\n :caption: dbschema/default.gel\n\n using extension ai;\n\n module default {\n abstract type Timestamped {\n required created_at: datetime {\n default := datetime_of_statement();\n };\n required updated_at: datetime {\n default := datetime_of_statement();\n };\n }\n\n type Deck extending Timestamped {\n required name: str;\n description: str;\n\n multi cards: Card {\n constraint exclusive;\n on target delete allow;\n };\n };\n\n type Card extending Timestamped {\n required order: int64;\n required front: str;\n required back: str;\n\n + deferred index ext::ai::index(embedding_model := 'text-embedding-3-small')\n + on (.front ++ ' ' ++ .back);\n }\n }\n\n\n.. edb:split-section::\n\n It's time to start running queries.\n\n Let's begin by creating the ``/fetch_similar`` endpoint we mentioned\n earlier. It's job is going to be to find 3 flashcards that are the most\n similar to the provided text query. We can use this endpoint to implement a\n \"recommended flashcards\" on the frontend.\n\n The AI extension contains a function called ``ext::ai::search(Type,\n embedding_vector)`` that we can use to do our fetch. Note that the second\n argument is an embedding vector, not a text query. To transform our text\n query into a vector, we will use the ``generate_embeddings`` function from\n the ``ai`` module of |Gel|'s Python binding.\n\n Gathered together, here are the modifications we need to do to the\n ``main.py`` function:\n\n\n .. code-block:: python-diff\n :caption: main.py\n\n import gel\n + import gel.ai\n\n from fastapi import FastAPI\n\n\n client = gel.create_async_client()\n\n app = FastAPI()\n\n\n + @app.get(\"/fetch_similar\")\n + async def fetch_similar_cards(query: str):\n + rag = await gel.ai.create_async_rag_client(client, model=\"gpt-4-turbo-preview\")\n + embedding_vector = await rag.generate_embeddings(\n + query, model=\"text-embedding-3-small\"\n + )\n\n + similar_cards = await client.query(\n + \"select ext::ai::search(Card, >$embedding_vector)\",\n + embedding_vector=embedding_vector,\n + )\n\n + return similar_cards\n\n\n.. edb:split-section::\n\n Let's test the endpoint to see that everything works the way we expect.\n\n\n .. code-block:: bash\n\n $ curl -X 'GET' \\\n 'http://localhost:8000/fetch_similar?query=the%20stinkiest%20cheese' \\\n -H 'accept: application/json'\n\n\n.. edb:split-section::\n\n Finally, let's create the second endpoint we mentioned, called\n ``/fetch_rag``. We'll be able to use this one to, for example, ask an LLM\n to quiz us on the contents of our deck.\n\n The RAG feature is represented in the Python binding with the ``query_rag``\n method of the ``GelRAG`` class. To use it, we're going to instantiate the\n class and call the method... And that's it!\n\n\n .. code-block:: python-diff\n :caption: main.py\n\n import gel\n import gel.ai\n\n from fastapi import FastAPI\n\n\n client = gel.create_async_client()\n\n app = FastAPI()\n\n\n @app.get(\"/fetch_similar\")\n async def fetch_similar_cards(query: str):\n rag = await gel.ai.create_async_rag_client(client, model=\"gpt-4-turbo-preview\")\n embedding_vector = await rag.generate_embeddings(\n query, model=\"text-embedding-3-small\"\n )\n\n similar_cards = await client.query(\n \"select ext::ai::search(Card, >$embedding_vector)\",\n embedding_vector=embedding_vector,\n )\n\n return similar_cards\n\n\n + @app.get(\"/fetch_rag\")\n + async def fetch_rag_response(query: str):\n + rag = await gel.ai.create_async_rag_client(client, model=\"gpt-4-turbo-preview\")\n + response = await rag.query_rag(\n + message=query,\n + context=gel.ai.QueryContext(query=\"select Card\"),\n + )\n + return response\n\n\n.. edb:split-section::\n\n Let's test the endpoint to see if it works:\n\n\n .. code-block:: bash\n\n $ curl -X 'GET' \\\n 'http://localhost:8000/fetch_rag?query=what%20cheese%20smells%20like%20feet' \\\n -H 'accept: application/json'\n\n\n\nCongratulations! We've now implemented AI features in our flashcards app.\nOf course, there's more to learn when it comes to using the AI extension.\nMake sure to check out the :ref:`Reference manual\n`, or build an LLM-powered search bot from the\nground up with the :ref:`FastAPI Gel AI tutorial\n`.\n" }, { "path": "docs/intro/quickstart/ai/index.rst", "content": ".. edb:env-switcher::\n\n=========\nAdding AI\n=========\n\n.. toctree::\n :maxdepth: 1\n\n fastapi\n" }, { "path": "docs/intro/quickstart/connecting/fastapi.rst", "content": ".. _ref_quickstart_fastapi_connecting:\n\n==========================\nConnecting to the database\n==========================\n\n.. edb:split-section::\n\n Before diving into the application, let's take a quick look at how to connect to the database from your code. We will intialize a client and use it to make a simple, static query to the database, and log the result to the console.\n\n .. note::\n\n Notice that the ``create_async_client`` function isn't being passed any connection details. With |Gel|, you do not need to come up with your own scheme for how to build the correct database connection credentials and worry about leaking them into your code. You simply use |Gel| \"projects\" for local development, and set the appropriate environment variables in your deployment environments, and the ``create_async_client`` function knows what to do!\n\n .. edb:split-point::\n\n .. code-block:: python\n :caption: ./test.py\n\n import gel\n import asyncio\n\n async def main():\n client = gel.create_async_client()\n result = await client.query_single(\"select 'Hello from Gel!';\")\n print(result)\n\n asyncio.run(main())\n\n .. code-block:: sh\n\n $ python test.py\n Hello from Gel!\n\n.. edb:split-section::\n\n In Python, we write EdgeQL queries directly as strings. This gives us the full power and expressiveness of EdgeQL while maintaining type safety through Gel's strict schema. Let's try inserting a few ``Deck`` objects into the database and then selecting them back.\n\n .. edb:split-point::\n\n .. code-block:: python-diff\n :caption: ./test.py\n\n import gel\n import asyncio\n\n async def main():\n client = gel.create_async_client()\n - result = await client.query_single(\"select 'Hello from Gel!';\")\n - print(result)\n + await client.query(\"\"\"\n + insert Deck { name := \"I am one\" }\n + \"\"\")\n +\n + await client.query(\"\"\"\n + insert Deck { name := \"I am two\" }\n + \"\"\")\n +\n + decks = await client.query(\"\"\"\n + select Deck {\n + id,\n + name\n + }\n + \"\"\")\n +\n + for deck in decks:\n + print(f\"ID: {deck.id}, Name: {deck.name}\")\n +\n + await client.query(\"delete Deck\")\n\n asyncio.run(main())\n\n .. code-block:: sh\n\n $ python test.py\n Hello from Gel!\n ID: f4cd3e6c-ea75-11ef-83ec-037350ea8a6e, Name: I am one\n ID: f4cf27ae-ea75-11ef-83ec-3f7b2fceab24, Name: I am two\n" }, { "path": "docs/intro/quickstart/connecting/index.rst", "content": ".. edb:env-switcher::\n\n==========================\nConnecting to the database\n==========================\n\n.. toctree::\n :maxdepth: 3\n :hidden:\n\n nextjs\n fastapi\n" }, { "path": "docs/intro/quickstart/connecting/nextjs.rst", "content": ".. _ref_quickstart_connecting:\n\n==========================\nConnecting to the database\n==========================\n\n.. edb:split-section::\n\n Before diving into the application, let's take a quick look at how to connect to the database from your code. We will intialize a client and use it to make a simple, static query to the database, and log the result to the console.\n\n .. note::\n\n Notice that the ``createClient`` function isn't being passed any connection details. With |Gel|, you do not need to come up with your own scheme for how to build the correct database connection credentials and worry about leaking them into your code. You simply use |Gel| \"projects\" for local development, and set the appropriate environment variables in your deployment environments, and the ``createClient`` function knows what to do!\n\n .. edb:split-point::\n\n .. code-block:: typescript\n :caption: ./test.ts\n\n import { createClient } from \"gel\";\n\n const client = createClient();\n\n async function main() {\n console.log(await client.query(\"select 'Hello from Gel!';\"));\n }\n\n main().then(\n () => process.exit(0),\n (err) => {\n console.error(err);\n process.exit(1);\n }\n );\n\n\n .. code-block:: sh\n\n $ npx tsx test.ts\n [ 'Hello from Gel!' ]\n\n.. edb:split-section::\n\n\n With TypeScript, there are three ways to run a query: use a string EdgeQL query, use the ``queries`` generator to turn a string of EdgeQL into a TypeScript function, or use the query builder API to build queries dynamically in a type-safe manner. In this tutorial, you will use the TypeScript query builder API.\n\n This query builder must be regenerated any time the schema changes, so a hook has been added to the ``gel.toml`` file to generate the query builder any time the schema is updated. Moving beyond this simple query, use the query builder API to insert a few ``Deck`` objects into the database, and then select them back.\n\n .. edb:split-point::\n\n .. code-block:: typescript-diff\n :caption: ./test.ts\n\n import { createClient } from \"gel\";\n + import e from \"@/dbschema/edgeql-js\";\n\n const client = createClient();\n\n async function main() {\n console.log(await client.query(\"select 'Hello from Gel!';\"));\n\n + await e.insert(e.Deck, { name: \"I am one\" }).run(client);\n +\n + await e.insert(e.Deck, { name: \"I am two\" }).run(client);\n +\n + const decks = await e\n + .select(e.Deck, () => ({\n + id: true,\n + name: true,\n + }))\n + .run(client);\n +\n + console.table(decks);\n +\n + await e.delete(e.Deck).run(client);\n }\n\n main().then(\n () => process.exit(0),\n (err) => {\n console.error(err);\n process.exit(1);\n }\n );\n\n .. code-block:: sh\n\n $ npx tsx test.ts\n [ 'Hello from Gel!' ]\n ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”\n ā”‚ (index) ā”‚ id ā”‚ name ā”‚\n ā”œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¼ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¼ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¤\n ā”‚ 0 ā”‚ 'f4cd3e6c-ea75-11ef-83ec-037350ea8a6e' ā”‚ 'I am one' ā”‚\n ā”‚ 1 ā”‚ 'f4cf27ae-ea75-11ef-83ec-3f7b2fceab24' ā”‚ 'I am two' ā”‚\n ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜\n\nNow that you know how to connect to the database, you will see that we have provided an initialized ``Client`` object in the ``/lib/gel.ts`` module. Throughout the rest of the tutorial, you will import this ``Client`` object and use it to make queries.\n" }, { "path": "docs/intro/quickstart/index.rst", "content": "==========\nQuickstart\n==========\n\n.. toctree::\n :maxdepth: 1\n :hidden:\n\n overview/index\n setup/index\n modeling/index\n connecting/index\n working/index\n inheritance/index\n ai/index\n" }, { "path": "docs/intro/quickstart/inheritance/fastapi.rst", "content": ".. _ref_quickstart_fastapi_inheritance:\n\n========================\nAdding shared properties\n========================\n\n.. edb:split-section::\n\n One common pattern in applications is to add shared properties to the schema that are used by multiple objects. For example, you might want to add a ``created_at`` and ``updated_at`` property to every object in your schema. You can do this by adding an abstract type and using it as a mixin for your other object types.\n\n .. code-block:: sdl-diff\n :caption: dbschema/default.gel\n\n module default {\n + abstract type Timestamped {\n + required created_at: datetime {\n + default := datetime_of_statement();\n + };\n + required updated_at: datetime {\n + default := datetime_of_statement();\n + };\n + }\n +\n - type Deck {\n + type Deck extending Timestamped {\n required name: str;\n description: str;\n\n cards := (\n select . y\n did you alter object type 'default::Card'? [y,n,l,c,b,s,q,?]\n > y\n did you alter object type 'default::Deck'? [y,n,l,c,b,s,q,?]\n > y\n Created /home/strinh/projects/flashcards/dbschema/migrations/00004-m1d2m5n.edgeql, id: m1d2m5n5ajkalyijrxdliioyginonqbtfzihvwdfdmfwodunszstya\n\n $ gel migrate\n Applying m1d2m5n5ajkalyijrxdliioyginonqbtfzihvwdfdmfwodunszstya (00004-m1d2m5n.edgeql)\n ... parsed\n ... applied\n\n.. edb:split-section::\n\n Update the ``get_decks`` query to sort the decks by ``updated_at`` in descending order.\n\n .. code-block:: python-diff\n :caption: main.py\n\n @app.get(\"/decks\", response_model=List[Deck])\n async def get_decks():\n decks = await client.query(\"\"\"\n select Deck {\n id,\n name,\n description,\n cards := (\n select .cards {\n id,\n front,\n back\n }\n order by .order\n )\n }\n + order by .updated_at desc\n \"\"\")\n return decks\n" }, { "path": "docs/intro/quickstart/inheritance/index.rst", "content": ".. edb:env-switcher::\n\n========================\nAdding shared properties\n========================\n\n.. toctree::\n :maxdepth: 3\n :hidden:\n\n nextjs\n fastapi\n\n" }, { "path": "docs/intro/quickstart/inheritance/nextjs.rst", "content": ".. _ref_quickstart_inheritance:\n\n========================\nAdding shared properties\n========================\n\n.. edb:split-section::\n\n One common pattern in applications is to add shared properties to the schema that are used by multiple objects. For example, you might want to add a ``created_at`` and ``updated_at`` property to every object in your schema. You can do this by adding an abstract type and using it as a mixin for your other object types.\n\n .. code-block:: sdl-diff\n :caption: dbschema/default.gel\n\n module default {\n + abstract type Timestamped {\n + required created_at: datetime {\n + default := datetime_of_statement();\n + };\n + required updated_at: datetime {\n + default := datetime_of_statement();\n + };\n + }\n +\n - type Deck {\n + type Deck extending Timestamped {\n required name: str;\n description: str;\n\n multi cards: Card {\n constraint exclusive;\n on target delete allow;\n };\n };\n\n - type Card {\n + type Card extending Timestamped {\n required order: int64;\n required front: str;\n required back: str;\n }\n }\n\n.. edb:split-section::\n\n Since you don't have historical data for when these objects were actually created or modified, the migration will fall back to the default values set in the ``Timestamped`` type.\n\n .. code-block:: sh\n\n $ npx gel migration create\n did you create object type 'default::Timestamped'? [y,n,l,c,b,s,q,?]\n > y\n did you alter object type 'default::Card'? [y,n,l,c,b,s,q,?]\n > y\n did you alter object type 'default::Deck'? [y,n,l,c,b,s,q,?]\n > y\n Created /home/strinh/projects/flashcards/dbschema/migrations/00004-m1d2m5n.edgeql, id: m1d2m5n5ajkalyijrxdliioyginonqbtfzihvwdfdmfwodunszstya\n\n $ npx gel migrate\n Applying m1d2m5n5ajkalyijrxdliioyginonqbtfzihvwdfdmfwodunszstya (00004-m1d2m5n.edgeql)\n ... parsed\n ... applied\n Generating query builder...\n Detected tsconfig.json, generating TypeScript files.\n To override this, use the --target flag.\n Run `npx @gel/generate --help` for full options.\n Introspecting database schema...\n Generating runtime spec...\n Generating cast maps...\n Generating scalars...\n Generating object types...\n Generating function types...\n Generating operators...\n Generating set impl...\n Generating globals...\n Generating index...\n Writing files to ./dbschema/edgeql-js\n Generation complete! šŸ¤˜\n\n.. edb:split-section::\n\n Update the ``getDecks`` query to sort the decks by ``updated_at`` in descending order.\n\n .. code-block:: typescript-diff\n :caption: app/queries.ts\n\n import { client } from \"@/lib/gel\";\n import e from \"@/dbschema/edgeql-js\";\n\n export async function getDecks() {\n const decks = await e.select(e.Deck, (deck) => ({\n id: true,\n name: true,\n description: true,\n cards: e.select(deck.cards, (card) => ({\n id: true,\n front: true,\n back: true,\n order_by: card.order,\n })),\n + order_by: {\n + expression: deck.updated_at,\n + direction: e.DESC,\n + },\n })).run(client);\n\n return decks;\n }\n\n.. edb:split-section::\n\n Now when you look at the data in the UI, you will see the new properties on each of your object types.\n\n .. image:: images/timestamped.png\n" }, { "path": "docs/intro/quickstart/modeling/fastapi.rst", "content": ".. _ref_quickstart_fastapi_modeling:\n\n=================\nModeling the data\n=================\n\n.. edb:split-section::\n\n The flashcards application has a simple data model, but it's interesting enough to utilize many unique features of the |Gel| schema language.\n\n Looking at the mock data in the example JSON file ``./deck-edgeql.json``, you can see this structure in the JSON. There is a ``Card`` class that describes a single flashcard, which contains two required string properties: ``front`` and ``back``. Each ``Deck`` object has zero or more ``Card`` objects in a list.\n\n .. code-block:: python\n\n from pydantic import BaseModel\n\n class CardBase(BaseModel):\n front: str\n back: str\n\n class Card(CardBase):\n id: str\n\n class DeckBase(BaseModel):\n name: str\n description: Optional[str] = None\n\n class Deck(DeckBase):\n id: str\n cards: List[Card]\n\n.. edb:split-section::\n\n Starting with this simple model, add these types to the :dotgel:`dbschema/default` schema file. As you can see, the types closely mirror the JSON mock data.\n\n Also of note, the link between ``Card`` and ``Deck`` objects creates a \"1-to-n\" relationship, where each ``Deck`` object has a link to zero or more ``Card`` objects. When you query the ``Deck.cards`` link, the cards will be unordered, so the ``Card`` type needs an explicit ``order`` property to allow sorting them at query time.\n\n By default, when you try to delete an object that is linked to another object, the database will prevent you from doing so. We want to support removing a ``Card``, so we define a deletion policy on the ``cards`` link that allows deleting the target of this link.\n\n .. code-block:: sdl-diff\n :caption: dbschema/default.gel\n\n module default {\n + type Card {\n + required order: int64;\n + required front: str;\n + required back: str;\n + };\n +\n + type Deck {\n + required name: str;\n + description: str;\n + multi cards: Card {\n + constraint exclusive;\n + on target delete allow;\n + };\n + };\n };\n\n.. edb:split-section::\n\n Congratulations! This first version of the data model's schema is *stored in a file on disk*. Now you need to signal the database to actually create types for ``Deck`` and ``Card`` in the database.\n\n To make |Gel| do that, you need to do two quick steps:\n\n 1. **Create a migration**: a \"migration\" is a file containing a set of low level instructions that define how the database schema should change. It records any additions, modifications, or deletions to your schema in a way that the database can understand.\n\n .. note::\n\n When you are changing existing schema, the CLI migration tool might ask questions to ensure that it understands your changes exactly. Since the existing schema was empty, the CLI will skip asking any questions and simply create the migration file.\n\n 2. **Apply the migration**: This executes the migration file on the database, instructing |Gel| to implement the recorded changes in the database. Essentially, this step updates the database structure to match your defined schema, ensuring that the ``Deck`` and ``Card`` types are created and ready for use.\n\n .. code-block:: sh\n\n $ uvx gel migration create\n Created ./dbschema/migrations/00001-m125ajr.edgeql, id: m125ajrbqp7ov36s7aniefxc376ofxdlketzspy4yddd3hrh4lxmla\n $ uvx gel migrate\n Applying m125ajrbqp7ov36s7aniefxc376ofxdlketzspy4yddd3hrh4lxmla (00001-m125ajr.edgeql)\n ... parsed\n ... applied\n\n\n.. edb:split-section::\n\n Take a look at the schema you've generated in the built-in database UI. Use this tool to visualize your data model and see the object types and links you've defined.\n\n .. edb:split-point::\n\n .. code-block:: sh\n\n $ uvx gel ui\n\n .. image:: images/schema-ui.png\n" }, { "path": "docs/intro/quickstart/modeling/index.rst", "content": ".. edb:env-switcher::\n\n=================\nModeling the data\n=================\n\n.. toctree::\n :maxdepth: 3\n :hidden:\n\n nextjs\n fastapi\n" }, { "path": "docs/intro/quickstart/modeling/nextjs.rst", "content": ".. _ref_quickstart_modeling:\n\n=================\nModeling the data\n=================\n\n.. edb:split-section::\n\n The flashcards application has a simple data model, but it's interesting enough to utilize many unique features of the |Gel| schema language.\n\n Looking at the mock data in the example JSON file ``./deck-edgeql.json``, you can see this structure in the JSON. There is a ``Card`` type that describes a single flashcard, which contains two required string properties: ``front`` and ``back``. Each ``Deck`` object has zero or more ``Card`` objects in an array.\n\n .. code-block:: typescript\n\n interface Card {\n front: string;\n back: string;\n }\n\n interface Deck {\n name: string;\n description: string | null;\n cards: Card[];\n }\n\n.. edb:split-section::\n\n Starting with this simple model, add these types to the :dotgel:`dbschema/default` schema file. As you can see, the types closely mirror the JSON mock data.\n\n Also of note, the link between ``Card`` and ``Deck`` objects creates a \"1-to-n\" relationship, where each ``Deck`` object has a link to zero or more ``Card`` objects. When you query the ``Deck.cards`` link, the cards will be unordered, so the ``Card`` type needs an explicit ``order`` property to allow sorting them at query time.\n\n By default, when you try to delete an object that is linked to another object, the database will prevent you from doing so. We want to support removing a ``Card``, so we define a deletion policy on the ``cards`` link that allows deleting the target of this link.\n\n .. code-block:: sdl-diff\n :caption: dbschema/default.gel\n\n module default {\n + type Card {\n + required order: int64;\n + required front: str;\n + required back: str;\n + };\n +\n + type Deck {\n + required name: str;\n + description: str;\n + multi cards: Card {\n + constraint exclusive;\n + on target delete allow;\n + };\n + };\n };\n\n.. edb:split-section::\n\n Congratulations! This first version of the data model's schema is *stored in a file on disk*. Now you need to signal the database to actually create types for ``Deck`` and ``Card`` in the database.\n\n To make |Gel| do that, you need to do two quick steps:\n\n 1. **Create a migration**: a \"migration\" is a file containing a set of low level instructions that define how the database schema should change. It records any additions, modifications, or deletions to your schema in a way that the database can understand.\n\n .. note::\n\n When you are changing existing schema, the CLI migration tool might ask questions to ensure that it understands your changes exactly. Since the existing schema was empty, the CLI will skip asking any questions and simply create the migration file.\n\n 2. **Apply the migration**: This executes the migration file on the database, instructing |Gel| to implement the recorded changes in the database. Essentially, this step updates the database structure to match your defined schema, ensuring that the ``Deck`` and ``Card`` types are created and ready for use.\n\n .. note::\n\n Notice that after the migration is applied, the CLI will automatically run the script to generate the query builder. This is a convenience feature that is enabled by the ``schema.update.after`` hook in the ``gel.toml`` file.\n\n .. code-block:: sh\n\n $ npx gel migration create\n Created ./dbschema/migrations/00001-m125ajr.edgeql, id: m125ajrbqp7ov36s7aniefxc376ofxdlketzspy4yddd3hrh4lxmla\n $ npx gel migrate\n Applying m125ajrbqp7ov36s7aniefxc376ofxdlketzspy4yddd3hrh4lxmla (00001-m125ajr.edgeql)\n ... parsed\n ... applied\n Generating query builder...\n Detected tsconfig.json, generating TypeScript files.\n To override this, use the --target flag.\n Run `npx @gel/generate --help` for full options.\n Introspecting database schema...\n Generating runtime spec...\n Generating cast maps...\n Generating scalars...\n Generating object types...\n Generating function types...\n Generating operators...\n Generating set impl...\n Generating globals...\n Generating index...\n Writing files to ./dbschema/edgeql-js\n Generation complete! šŸ¤˜\n\n\n.. edb:split-section::\n\n Take a look at the schema you've generated in the built-in database UI. Use this tool to visualize your data model and see the object types and links you've defined.\n\n .. edb:split-point::\n\n .. code-block:: sh\n\n $ npx gel ui\n\n .. image:: images/schema-ui.png\n" }, { "path": "docs/intro/quickstart/overview/fastapi.rst", "content": ".. _ref_quickstart_fastapi:\n\n==========\nQuickstart\n==========\n\nWelcome to the quickstart tutorial! In this tutorial, you will update a FastAPI\nbackend for a Flashcards application to use |Gel| as your data layer. The\napplication will let users build and manage their own study decks, with each\nflashcard featuring customizable text on both sides - making it perfect for\nstudying, memorization practice, or creating educational games.\n\nDon't worry if you're new to |Gel| - you will be up and running with a working\nFastAPI backend and a local |Gel| database in just about **5 minutes**. From\nthere, you will replace the static mock data with a |Gel| powered data layer in\nroughly 30-45 minutes.\n\nBy the end of this tutorial, you will be comfortable with:\n\n* Creating and updating a database schema\n* Running migrations to evolve your data\n* Writing EdgeQL queries\n* Building an app backed by |Gel|\n\n\nFeatures of the flashcards app\n------------------------------\n\n* Create, edit, and delete decks\n* Add/remove cards with front/back content\n* Clean, type-safe schema with |Gel|\n\nRequirements\n------------\n\nBefore you start, you need:\n\n* Basic familiarity with Python and FastAPI\n* Python 3.8+ on a Unix-like OS (Linux, macOS, or WSL)\n* A code editor you love\n\nWhy |Gel| for FastAPI?\n----------------------\n\n* **Type Safety**: Catch data errors before runtime\n* **Rich Modeling**: Use object types and links to model relations\n* **Modern Tooling**: Python-friendly schemas and migrations\n* **Performance**: Efficient queries for complex data\n* **Developer Experience**: An intuitive query language (EdgeQL)\n\nNeed Help?\n----------\n\nIf you run into issues while following this tutorial:\n\n- Check the `Gel documentation `_\n- Visit our `community Discord `_\n- File an issue on `GitHub `_\n" }, { "path": "docs/intro/quickstart/overview/index.rst", "content": ".. edb:env-switcher::\n\n========\nOverview\n========\n\n.. toctree::\n :maxdepth: 3\n :hidden:\n\n nextjs\n fastapi\n" }, { "path": "docs/intro/quickstart/overview/nextjs.rst", "content": ".. _gel-js-quickstart:\n.. _ref_quickstart:\n\n==========\nQuickstart\n==========\n\nWelcome to the quickstart tutorial! In this tutorial, you will update a simple Next.js application to use |Gel| as your data layer. The application will let users build and manage their own study decks, with each flashcard featuring customizable text on both sides - making it perfect for studying, memorization practice, or creating educational games.\n\nDon't worry if you're new to |Gel| - you will be up and running with a working Next.js application and a local |Gel| database in just about **5 minutes**. From there, you will replace the static mock data with a |Gel| powered data layer in roughly 30-45 minutes.\n\nBy the end of this tutorial, you will be comfortable with:\n\n* Creating and updating a database schema\n* Running migrations to evolve your data\n* Writing EdgeQL queries in text and via a TypeScript query builder\n* Building an app backed by |Gel|\n\nFeatures of the flashcards app\n------------------------------\n\n* Create, edit, and delete decks\n* Add/remove cards with front/back content\n* Simple Next.js + Tailwind UI\n* Clean, type-safe schema with |Gel|\n\nRequirements\n------------\n\nBefore you start, you need:\n\n* Basic familiarity with TypeScript, Next.js, and React\n* Node.js 20+ on a Unix-like OS (Linux, macOS, or WSL)\n* A code editor you love\n\nWhy |Gel| for Next.js?\n----------------------\n\n* **Type Safety**: Catch data errors before runtime\n* **Rich Modeling**: Use object types and links to model relations\n* **Modern Tooling**: TypeScript-friendly schemas and migrations\n* **Performance**: Efficient queries for complex data\n* **Developer Experience**: An intuitive query language (EdgeQL)\n\nNeed Help?\n----------\n\nIf you run into issues while following this tutorial:\n\n* Check the `Gel documentation `_\n* Visit our `community Discord `_\n* File an issue on `GitHub `_\n" }, { "path": "docs/intro/quickstart/setup/fastapi.rst", "content": ".. _ref_quickstart_fastapi_setup:\n\n===========================\nSetting up your environment\n===========================\n\n.. edb:split-section::\n\n Use git to clone the `FastAPI starter template `_ into a new directory called ``flashcards``. This will create a fully configured FastAPI project and a local |Gel| instance with an empty schema. You will see the database instance being created and the project being initialized. You are now ready to start building the application.\n\n .. code-block:: sh\n\n $ git clone \\\n git@github.com:geldata/quickstart-fastapi.git \\\n flashcards\n $ cd flashcards\n $ python -m venv venv\n $ source venv/bin/activate # or venv\\Scripts\\activate on Windows\n $ pip install -r requirements.txt\n $ uvx gel project init\n\n.. edb:split-section::\n\n Explore the empty database by starting our REPL from the project root.\n\n .. code-block:: sh\n\n $ uvx gel\n\n.. edb:split-section::\n\n Try the following queries which will work without any schema defined.\n\n .. code-block:: edgeql-repl\n\n db> select 42;\n {42}\n db> select sum({1, 2, 3});\n {6}\n db> with cards := {\n ... (\n ... front := \"What is the highest mountain in the world?\",\n ... back := \"Mount Everest\",\n ... ),\n ... (\n ... front := \"Which ocean contains the deepest trench on Earth?\",\n ... back := \"The Pacific Ocean\",\n ... ),\n ... }\n ... select cards order by random() limit 1;\n {\n (\n front := \"What is the highest mountain in the world?\",\n back := \"Mount Everest\",\n )\n }\n\n.. edb:split-section::\n\n Fun! You will create a proper data model for the application in the next step, but for now, take a look around the project we have. Here are the files that integrate |Gel|:\n\n - ``gel.toml``: The configuration file for the |Gel| project instance. Notice that we have a ``hooks.migration.apply.after`` hook that will run ``uvx gel-py`` after migrations are applied. This will run the code generator that you will use later to get fully type-safe queries you can run from your FastAPI backend. More details on that to come!\n - ``dbschema/``: This directory contains the schema for the database, and later supporting files like migrations, and generated code.\n - :dotgel:`dbschema/default`: The default schema file that you'll use to define your data model. It is empty for now, but you'll add your data model to this file in the next step.\n\n .. tabs::\n\n .. code-tab:: toml\n :caption: gel.toml\n\n [instance]\n server-version = \"6.11\"\n\n [hooks]\n schema.update.after = \"uvx gel-py\"\n\n .. code-tab:: sdl\n :caption: dbschema/default.gel\n\n module default {\n\n }\n" }, { "path": "docs/intro/quickstart/setup/index.rst", "content": ".. edb:env-switcher::\n\n===========================\nSetting up your environment\n===========================\n\n.. toctree::\n :maxdepth: 3\n :hidden:\n\n nextjs\n fastapi\n" }, { "path": "docs/intro/quickstart/setup/nextjs.rst", "content": ".. _ref_quickstart_setup:\n\n===========================\nSetting up your environment\n===========================\n\n.. edb:split-section::\n\n Use git to clone `the Next.js starter template `_ into a new directory called ``flashcards``. This will create a fully configured Next.js project and a local |Gel| instance with an empty schema. You will see the database instance being created and the project being initialized. You are now ready to start building the application.\n\n .. code-block:: sh\n\n $ git clone \\\n git@github.com:geldata/quickstart-nextjs.git \\\n flashcards\n $ cd flashcards\n $ npm install\n $ npx gel project init\n\n\n.. edb:split-section::\n\n Explore the empty database by starting our REPL from the project root.\n\n .. code-block:: sh\n\n $ npx gel\n\n.. edb:split-section::\n\n Try the following queries which will work without any schema defined.\n\n .. code-block:: edgeql-repl\n\n db> select 42;\n {42}\n db> select sum({1, 2, 3});\n {6}\n db> with cards := {\n ... (\n ... front := \"What is the highest mountain in the world?\",\n ... back := \"Mount Everest\",\n ... ),\n ... (\n ... front := \"Which ocean contains the deepest trench on Earth?\",\n ... back := \"The Pacific Ocean\",\n ... ),\n ... }\n ... select cards order by random() limit 1;\n {\n (\n front := \"What is the highest mountain in the world?\",\n back := \"Mount Everest\",\n )\n }\n\n.. edb:split-section::\n\n Fun! You will create a proper data model for the application in the next step, but for now, take a look around the project you've just created. Most of the project files will be familiar if you've worked with Next.js before. Here are the files that integrate |Gel|:\n\n - ``gel.toml``: The configuration file for the |Gel| project instance. Notice that we have a ``hooks.migration.apply.after`` hook that will run ``npx @gel/generate edgeql-js`` after migrations are applied. This will generate the query builder code that you'll use to interact with the database. More details on that to come!\n - ``dbschema/``: This directory contains the schema for the database, and later supporting files like migrations, and generated code.\n - :dotgel:`dbschema/default`: The default schema file that you'll use to define your data model. It is empty for now, but you'll add your data model to this file in the next step.\n - ``lib/gel.ts``: A utility module that exports the |Gel| client, which you'll use to interact with the database.\n\n .. tabs::\n\n .. code-tab:: toml\n :caption: gel.toml\n\n [instance]\n server-version = \"6.11\"\n\n [hooks]\n schema.update.after = \"npx @gel/generate edgeql-js\"\n\n .. code-tab:: sdl\n :caption: dbschema/default.gel\n\n module default {\n\n }\n\n .. code-tab:: typescript\n :caption: lib/gel.ts\n\n import { createClient } from \"gel\";\n\n export const client = createClient();\n" }, { "path": "docs/intro/quickstart/working/fastapi.rst", "content": ".. _ref_quickstart_fastapi_working:\n\n=====================\nWorking with the data\n=====================\n\nIn this section, you will update the existing FastAPI application to use |Gel| to store and query data, instead of a JSON file. Having a working application with mock data allows you to focus on learning how |Gel| works, without getting bogged down by the details of the application.\n\nBulk importing of data\n======================\n\n.. edb:split-section::\n\n First, update the imports and Pydantic models to use UUID instead of string for ID fields, since this is what |Gel| returns. You also need to initialize the |Gel| client and import the asyncio module to work with async functions.\n\n .. code-block:: python-diff\n :caption: main.py\n\n from fastapi import FastAPI, HTTPException\n from pydantic import BaseModel\n from typing import List, Optional\n - import json\n - from pathlib import Path\n + from uuid import UUID\n + from gel import create_async_client\n + import asyncio\n\n app = FastAPI(title=\"Flashcards API\")\n\n # Pydantic models\n class CardBase(BaseModel):\n front: str\n back: str\n\n class Card(CardBase):\n - id: str\n + id: UUID\n\n class DeckBase(BaseModel):\n name: str\n description: Optional[str] = None\n\n class DeckCreate(DeckBase):\n cards: List[CardBase]\n\n class Deck(DeckBase):\n - id: str\n + id: UUID\n cards: List[Card]\n\n - DATA_DIR = Path(__file__).parent / \"data\"\n - DECKS_FILE = DATA_DIR / \"decks.json\"\n + client = create_async_client()\n\n\n.. edb:split-section::\n\n Next, update the deck import operation to use |Gel| to create the deck and cards. The operation creates cards first, then creates a deck with links to the cards. Finally, it fetches the newly created deck with all required fields.\n\n .. note::\n\n Notice the ``{ ** }`` in the query. This is a shorthand for selecting all fields of the object. It's useful when you want to return the entire object without specifying each field. In our case, we want to return the entire deck object with all the nested fields.\n\n .. code-block:: python-diff\n :caption: main.py\n\n from fastapi import FastAPI, HTTPException\n from pydantic import BaseModel\n from typing import List, Optional\n from uuid import UUID\n from gel import create_async_client\n import asyncio\n\n app = FastAPI(title=\"Flashcards API\")\n\n # Pydantic models\n class CardBase(BaseModel):\n front: str\n back: str\n\n class Card(CardBase):\n id: UUID\n\n class DeckBase(BaseModel):\n name: str\n description: Optional[str] = None\n\n class DeckCreate(DeckBase):\n cards: List[CardBase]\n\n class Deck(DeckBase):\n id: UUID\n cards: List[Card]\n\n client = create_client()\n\n - DATA_DIR.mkdir(exist_ok=True)\n - if not DECKS_FILE.exists():\n - DECKS_FILE.write_text(\"[]\")\n\n - def read_decks() -> List[Deck]:\n - content = DECKS_FILE.read_text()\n - data = json.loads(content)\n - return [Deck(**deck) for deck in data]\n -\n - def write_decks(decks: List[Deck]) -> None:\n - data = [deck.model_dump() for deck in decks]\n - DECKS_FILE.write_text(json.dumps(data, indent=2))\n\n @app.post(\"/decks/import\", response_model=Deck)\n async def import_deck(deck: DeckCreate):\n - decks = read_decks()\n - new_deck = Deck(\n - id=str(uuid.uuid4()),\n - name=deck.name,\n - description=deck.description,\n - cards=[Card(id=str(uuid.uuid4()), **card.model_dump())\n - for card in deck.cards]\n - )\n - decks.append(new_deck)\n - write_decks(decks)\n - return new_deck\n + card_ids = []\n + for i, card in enumerate(deck.cards):\n + created_card = await client.query_single(\"\"\"\n + insert Card {\n + front := $front,\n + back := $back,\n + order := $order\n + }\n + \"\"\", front=card.front, back=card.back, order=i)\n + card_ids.append(created_card.id)\n +\n + new_deck = await client.query_single(\"\"\"\n + select(\n + insert Deck {\n + name := $name,\n + description := $description,\n + cards := (\n + select Card\n + filter contains(>$card_ids, .id)\n + )\n + }\n + ) { ** }\n + \"\"\", name=deck.name, description=deck.description,\n + card_ids=card_ids)\n +\n + return new_deck\n\n.. edb:split-section::\n\n The above works but isn't atomic - if any single query fails, you could end up with partial data. Let's wrap it in a transaction:\n\n .. code-block:: python-diff\n :caption: main.py\n\n @app.post(\"/decks/import\", response_model=Deck)\n async def import_deck(deck: DeckCreate):\n + async for tx in client.transaction():\n + async with tx:\n card_ids = []\n for i, card in enumerate(deck.cards):\n - created_card = await client.query_single(\n + created_card = await tx.query_single(\n \"\"\"\n insert Card {\n front := $front,\n back := $back,\n order := $order\n }\n \"\"\",\n front=card.front,\n back=card.back,\n order=i,\n )\n card_ids.append(created_card.id)\n\n - new_deck = await client.query_single(\"\"\"\n + new_deck = await tx.query_single(\"\"\"\n select(\n insert Deck {\n name := $name,\n description := $description,\n cards := (\n select Card\n filter .id IN array_unpack(>$card_ids)\n )\n }\n ) { ** }\n \"\"\",\n name=deck.name,\n description=deck.description,\n card_ids=card_ids,\n )\n\n return new_deck\n\n.. edb:split-section::\n\n One of the most powerful features of EdgeQL is the ability to compose complex queries in a way that is both readable and efficient. Use this super-power to create a single query that inserts the deck and cards, along with their links, in one efficient query.\n\n This new query uses a ``for`` expression to iterate over the set of cards, and sets the ``Deck.cards`` link to the result of inserting each card. This is logically equivalent to the previous approach, but is more efficient since it inserts the deck and cards in a single query.\n\n .. code-block:: python-diff\n :caption: main.py\n\n @app.post(\"/decks/import\", response_model=Deck)\n async def import_deck(deck: DeckCreate):\n - async for tx in client.transaction():\n - async with tx:\n - card_ids = []\n - for i, card in enumerate(deck.cards):\n - created_card = await tx.query_single(\n - \"\"\"\n - insert Card {\n - front := $front,\n - back := $back,\n - order := $order\n - }\n - \"\"\",\n - front=card.front,\n - back=card.back,\n - order=i,\n - )\n - card_ids.append(created_card.id)\n -\n - new_deck = await client.query_single(\"\"\"\n - select(\n - insert Deck {\n - name := $name,\n - description := $description,\n - cards := (\n - select Card\n - filter .id IN array_unpack(>$card_ids)\n - )\n - }\n - ) { ** }\n - \"\"\",\n - name=deck.name,\n - description=deck.description,\n - card_ids=card_ids,\n - )\n + cards_data = [(c.front, c.back, i) for i, c in enumerate(deck.cards)]\n +\n + new_deck = await client.query_single(\"\"\"\n + select(\n + with cards := >>$cards_data\n + insert Deck {\n + name := $name,\n + description := $description,\n + cards := (\n + for card in array_unpack(cards)\n + insert Card {\n + front := card.0,\n + back := card.1,\n + order := card.2\n + }\n + )\n + }\n + ) { ** }\n + \"\"\", name=deck.name, description=deck.description,\n + cards_data=cards_data)\n\n return new_deck\n\nUpdating data\n=============\n\n.. edb:split-section::\n\n Next, update the deck operations. The update operation needs to handle partial updates of name and description:\n\n .. code-block:: python-diff\n :caption: main.py\n\n @app.put(\"/decks/{deck_id}\", response_model=Deck)\n async def update_deck(deck_id: UUID, deck_update: DeckBase):\n - decks = read_decks()\n - deck = next((deck for deck in decks if deck.id == deck_id), None)\n - if not deck:\n - raise HTTPException(status_code=404, detail=\"Deck not found\")\n -\n - deck.name = deck_update.name\n - deck.description = deck_update.description\n - write_decks(decks)\n - return deck\n + # Build update sets based on provided fields\n + sets = []\n + params = {\"id\": deck_id}\n +\n + if deck_update.name is not None:\n + sets.append(\"name := $name\")\n + params[\"name\"] = deck_update.name\n +\n + if deck_update.description is not None:\n + sets.append(\"description := $description\")\n + params[\"description\"] = deck_update.description\n +\n + if not sets:\n + return await get_deck(deck_id)\n +\n + updated_deck = await client.query(f\"\"\"\n + with updated := (\n + update Deck\n + filter .id = $id\n + set {{ {', '.join(sets)} }}\n + )\n + select updated {{ ** }}\n + \"\"\", **params)\n +\n + if not updated_deck:\n + raise HTTPException(status_code=404, detail=\"Deck not found\")\n +\n + return updated_deck\n\n\nAdding linked data\n==================\n\n.. edb:split-section::\n\n Now, update the add card operation to use |Gel|. This operation will insert a new ``Card`` object and update the ``Deck.cards`` set to include the new ``Card`` object. Notice that the ``order`` property is set by selecting the maximum ``order`` property of this ``Deck.cards`` set and incrementing it by 1.\n\n The syntax for adding an object to a set of links is ``{ \"+=\": object }``. You can think of this as a shortcut for setting the link set to the current set plus the new object.\n\n .. code-block:: python-diff\n :caption: main.py\n\n @app.post(\"/decks/{deck_id}/cards\", response_model=Card)\n async def add_card(deck_id: UUID, card: CardBase):\n - decks = read_decks()\n - deck = next((deck for deck in decks if deck.id == deck_id), None)\n - if not deck:\n - raise HTTPException(status_code=404, detail=\"Deck not found\")\n -\n - new_card = Card(id=str(uuid.uuid4()), **card.model_dump())\n - deck.cards.append(new_card)\n - write_decks(decks)\n - return new_card\n + new_card = await client.query_single(\n + \"\"\"\n + with\n + deck := (select Deck filter .id = $id),\n + order := (max(deck.cards.order) + 1),\n + new_card := (\n + insert Card {\n + front := $front,\n + back := $back,\n + order := order,\n + }\n + ),\n + updated := (\n + update deck\n + set {\n + cards += new_card\n + }\n + ),\n + select new_card { ** }\n + \"\"\",\n + id=deck_id,\n + front=card.front,\n + back=card.back,\n + )\n +\n + if not new_card:\n + raise HTTPException(status_code=404, detail=\"Deck not found\")\n +\n + return new_card\n\n\nDeleting linked data\n====================\n\n.. edb:split-section::\n\n As the next step, update the card deletion operation to use |Gel| to remove a card from a deck:\n\n .. code-block:: python-diff\n :caption: main.py\n\n @app.delete(\"/cards/{card_id}\")\n async def delete_card(card_id: str):\n - decks = read_decks()\n - deck = next((deck for deck in decks if deck.id == deck_id), None)\n - if not deck:\n - raise HTTPException(status_code=404, detail=\"Deck not found\")\n -\n - deck.cards = [card for card in deck.cards if card.id != card_id]\n - write_decks(decks)\n + deleted = await client.query_single(\"\"\"\n + delete Card filter .id = $card_id\n + \"\"\", card_id=card_id)\n +\n + if not deleted:\n + raise HTTPException(status_code=404, detail=\"Card not found\")\n +\n return {\"message\": \"Card deleted\"}\n\nQuerying data\n=============\n\n.. edb:split-section::\n\n Finally, update the query endpoints to fetch data from |Gel|:\n\n .. code-block:: python-diff\n :caption: main.py\n\n @app.get(\"/decks\", response_model=List[Deck])\n async def get_decks():\n - return read_decks()\n + decks = await client.query(\"\"\"\n + select Deck {\n + id,\n + name,\n + description,\n + cards := (\n + select .cards {\n + id,\n + front,\n + back\n + }\n + order by .order\n + )\n + }\n + \"\"\")\n + return decks\n\n @app.get(\"/decks/{deck_id}\", response_model=Deck)\n async def get_deck(deck_id: UUID):\n - decks = read_decks()\n - deck = next((deck for deck in decks if deck.id == deck_id), None)\n - if not deck:\n - raise HTTPException(status_code=404, detail=f\"Deck with id {deck_id} not found\")\n - return deck\n + deck = await client.query_single(\"\"\"\n + select Deck {\n + id,\n + name,\n + description,\n + cards := (\n + select .cards {\n + id,\n + front,\n + back\n + }\n + order by .order\n + )\n + }\n + filter .id = $id\n + \"\"\", id=deck_id)\n +\n + if not deck:\n + raise HTTPException(\n + status_code=404,\n + detail=f\"Deck with id {deck_id} not found\"\n + )\n +\n + return deck\n\n.. edb:split-section::\n\n You can now run your FastAPI application with:\n\n .. code-block:: sh\n\n $ uvicorn main:app --reload\n\n.. edb:split-section::\n\n The API documentation will be available at http://localhost:8000/docs. You can use this interface to test your endpoints and import the sample flashcard deck.\n\n .. image:: images/flashcards-api.png\n" }, { "path": "docs/intro/quickstart/working/index.rst", "content": ".. edb:env-switcher::\n\n=====================\nWorking with the data\n=====================\n\n.. toctree::\n :maxdepth: 3\n :hidden:\n\n nextjs\n fastapi\n" }, { "path": "docs/intro/quickstart/working/nextjs.rst", "content": ".. _ref_quickstart_working:\n\n=====================\nWorking with the data\n=====================\n\nIn this section, you will update the existing application to use |Gel| to store and query data, instead of a static JSON file. Having a working application with mock data allows you to focus on learning how |Gel| works, without getting bogged down by the details of the application.\n\nBulk importing of data\n======================\n\n.. edb:split-section::\n\n Begin by updating the server action to import a deck with cards. Loop through each card in the deck and insert it, building an array of IDs as you go. This array of IDs will be used to set the ``cards`` link on the ``Deck`` object after all cards have been inserted.\n\n The array of card IDs is initially an array of strings. To satisfy the |Gel| type system, which expects the ``id`` property of ``Card`` objects to be a ``uuid`` rather than a ``str``, you need to cast the array of strings to an array of UUIDs. Use the ``e.literal(e.array(e.uuid), cardIds)`` function to perform this casting.\n\n The function ``e.contains(cardIdsLiteral, c.id)`` from our standard library checks if a value is present in an array and returns a boolean. When inserting the ``Deck`` object, set the ``cards`` to the result of selecting only the ``Card`` objects whose ``id`` is included in the ``cardIds`` array.\n\n .. code-block:: typescript-diff\n :caption: app/actions.ts\n\n \"use server\";\n\n - import { readFile, writeFile } from \"node:fs/promises\";\n + import { client } from \"@/lib/gel\";\n + import e from \"@/dbschema/edgeql-js\";\n import { revalidatePath } from \"next/cache\";\n - import { RawJSONDeck, Deck } from \"@/lib/models\";\n + import { RawJSONDeck } from \"@/lib/models\";\n\n export async function importDeck(formData: FormData) {\n const file = formData.get(\"file\") as File;\n const rawDeck = JSON.parse(await file.text()) as RawJSONDeck;\n const deck = {\n ...rawDeck,\n - id: crypto.randomUUID(),\n - cards: rawDeck.cards.map((card) => ({\n + cards: rawDeck.cards.map((card, index) => ({\n ...card,\n - id: crypto.randomUUID(),\n + order: index,\n })),\n };\n -\n - const existingDecks = JSON.parse(\n - await readFile(\"./decks.json\", \"utf-8\")\n - ) as Deck[];\n -\n - await writeFile(\n - \"./decks.json\",\n - JSON.stringify([...existingDecks, deck], null, 2)\n - );\n + const cardIds: string[] = [];\n + for (const card of deck.cards) {\n + const createdCard = await e\n + .insert(e.Card, {\n + front: card.front,\n + back: card.back,\n + order: card.order,\n + })\n + .run(client);\n +\n + cardIds.push(createdCard.id);\n + }\n +\n + const cardIdsLiteral = e.literal(e.array(e.uuid), cardIds);\n +\n + await e.insert(e.Deck, {\n + name: deck.name,\n + description: deck.description,\n + cards: e.select(e.Card, (c) => ({\n + filter: e.contains(cardIdsLiteral, c.id),\n + })),\n + }).run(client);\n\n revalidatePath(\"/\");\n }\n\n.. edb:split-section::\n\n This works, but you might notice that it is not atomic. For instance, if one of the ``Card`` objects fails to insert, the entire operation will fail and the ``Deck`` will not be inserted, but some data will still linger. To make this operation atomic, update the ``importDeck`` action to use a transaction.\n\n .. code-block:: typescript-diff\n :caption: app/actions.ts\n\n \"use server\";\n\n import { client } from \"@/lib/gel\";\n import e from \"@/dbschema/edgeql-js\";\n import { revalidatePath } from \"next/cache\";\n import { RawJSONDeck } from \"@/lib/models\";\n\n export async function importDeck(formData: FormData) {\n const file = formData.get(\"file\") as File;\n const rawDeck = JSON.parse(await file.text()) as RawJSONDeck;\n const deck = {\n ...rawDeck,\n cards: rawDeck.cards.map((card, index) => ({\n ...card,\n order: index,\n })),\n };\n + await client.transaction(async (tx) => {\n const cardIds: string[] = [];\n for (const card of deck.cards) {\n const createdCard = await e\n .insert(e.Card, {\n front: card.front,\n back: card.back,\n order: card.order,\n })\n - .run(client);\n + .run(tx);\n\n cardIds.push(createdCard.id);\n }\n\n const cardIdsLiteral = e.literal(e.array(e.uuid), cardIds);\n\n await e.insert(e.Deck, {\n name: deck.name,\n description: deck.description,\n cards: e.select(e.Card, (c) => ({\n filter: e.contains(cardIdsLiteral, c.id),\n })),\n - }).run(client);\n + }).run(tx);\n + });\n\n revalidatePath(\"/\");\n }\n\n.. edb:split-section::\n\n You might think this is as good as it gets, and many ORMs will create a similar set of queries. However, with the query builder, you can improve this by crafting a single query that inserts the ``Deck`` and ``Card`` objects, along with their links, in one efficient query.\n\n The first thing to notice is that the ``e.params`` function is used to define parameters for your query instead of embedding literal values directly. This approach eliminates the need for casting, as was necessary with the ``cardIds`` array. By defining the ``cards`` parameter as an array of tuples, you ensure full type safety with both TypeScript and the database.\n\n Another key feature of this query builder expression is the ``e.for(e.array_unpack(params.cards), (card) => {...})`` construct. This expression converts the array of tuples into a set of tuples and generates a set containing an expression for each element. Essentially, you assign the ``Deck.cards`` set of ``Card`` objects to the result of inserting each element from the ``cards`` array. This is similar to what you were doing before by selecting all ``Card`` objects by their ``id``, but is more efficient since you are inserting the ``Deck`` and all ``Card`` objects in one query.\n\n .. code-block:: typescript-diff\n :caption: app/actions.ts\n\n \"use server\";\n\n import { client } from \"@/lib/gel\";\n import e from \"@/dbschema/edgeql-js\";\n import { revalidatePath } from \"next/cache\";\n import { RawJSONDeck } from \"@/lib/models\";\n\n export async function importDeck(formData: FormData) {\n const file = formData.get(\"file\") as File;\n const rawDeck = JSON.parse(await file.text()) as RawJSONDeck;\n const deck = {\n ...rawDeck,\n cards: rawDeck.cards.map((card, index) => ({\n ...card,\n order: index,\n })),\n };\n - await client.transaction(async (tx) => {\n - const cardIds: string[] = [];\n - for (const card of deck.cards) {\n - const createdCard = await e\n - .insert(e.Card, {\n - front: card.front,\n - back: card.back,\n - order: card.order,\n - })\n - .run(tx);\n -\n - cardIds.push(createdCard.id);\n - }\n -\n - const cardIdsLiteral = e.literal(e.array(e.uuid), cardIds);\n -\n - await e.insert(e.Deck, {\n - name: deck.name,\n - description: deck.description,\n - cards: e.select(e.Card, (c) => ({\n - filter: e.contains(cardIdsLiteral, c.id),\n - })),\n - }).run(tx);\n - });\n + await e\n + .params(\n + {\n + name: e.str,\n + description: e.optional(e.str),\n + cards: e.array(e.tuple({ front: e.str, back: e.str, order: e.int64 })),\n + },\n + (params) =>\n + e.insert(e.Deck, {\n + name: params.name,\n + description: params.description,\n + cards: e.for(e.array_unpack(params.cards), (card) =>\n + e.insert(e.Card, {\n + front: card.front,\n + back: card.back,\n + order: card.order,\n + })\n + ),\n + })\n + )\n + .run(client, deck);\n\n revalidatePath(\"/\");\n }\n\nUpdating data\n=============\n\n.. edb:split-section::\n\n Next, you will update the Server Actions for each ``Deck`` object: ``updateDeck``, ``addCard``, and ``deleteCard``. Start with ``updateDeck``, which is the most complex because it is dynamic. You can set either the ``title`` or ``description`` fields in an update. Use the dynamic nature of the query builder to generate separate queries based on which fields are present in the form data.\n\n This may seem a bit intimidating at first, but the key to making this query dynamic is the ``nameSet`` and ``descriptionSet`` variables. These variables conditionally add the ``name`` or ``description`` fields to the ``set`` parameter of the ``update`` call.\n\n .. code-block:: typescript-diff\n :caption: app/deck/[id]/actions.ts\n\n \"use server\";\n\n import { revalidatePath } from \"next/cache\";\n import { readFile, writeFile } from \"node:fs/promises\";\n + import { client } from \"@/lib/gel\";\n + import e from \"@/dbschema/edgeql-js\";\n import { Deck } from \"@/lib/models\";\n\n export async function updateDeck(formData: FormData) {\n const id = formData.get(\"id\");\n const name = formData.get(\"name\");\n const description = formData.get(\"description\");\n\n if (\n typeof id !== \"string\" ||\n (typeof name !== \"string\" &&\n typeof description !== \"string\")\n ) {\n return;\n }\n\n - const decks = JSON.parse(\n - await readFile(\"./decks.json\", \"utf-8\")\n - ) as Deck[];\n - decks[index].name = name ?? decks[index].name;\n + const nameSet = typeof name === \"string\" ? { name } : {};\n - decks[index].description = description ?? decks[index].description;\n + const descriptionSet =\n + typeof description === \"string\" ? { description: description || null } : {};\n\n + await e\n + .update(e.Deck, (d) => ({\n + filter_single: e.op(d.id, \"=\", e.uuid(id)),\n + set: {\n + ...nameSet,\n + ...descriptionSet,\n + },\n + })).run(client);\n - await writeFile(\"./decks.json\", JSON.stringify(decks, null, 2));\n revalidatePath(`/deck/${id}`);\n }\n\n export async function addCard(formData: FormData) {\n const deckId = formData.get(\"deckId\");\n const front = formData.get(\"front\");\n const back = formData.get(\"back\");\n\n if (\n typeof deckId !== \"string\" ||\n typeof front !== \"string\" ||\n typeof back !== \"string\"\n ) {\n return;\n }\n\n const decks = JSON.parse(await readFile(\"./decks.json\", \"utf-8\")) as Deck[];\n\n const deck = decks.find((deck) => deck.id === deckId);\n if (!deck) {\n return;\n }\n\n deck.cards.push({ front, back, id: crypto.randomUUID() });\n await writeFile(\"./decks.json\", JSON.stringify(decks, null, 2));\n\n revalidatePath(`/deck/${deckId}`);\n }\n\n export async function deleteCard(formData: FormData) {\n const cardId = formData.get(\"cardId\");\n\n if (typeof cardId !== \"string\") {\n return;\n }\n\n const decks = JSON.parse(await readFile(\"./decks.json\", \"utf-8\")) as Deck[];\n const deck = decks.find((deck) => deck.cards.some((card) => card.id === cardId));\n if (!deck) {\n return;\n }\n\n deck.cards = deck.cards.filter((card) => card.id !== cardId);\n await writeFile(\"./decks.json\", JSON.stringify(decks, null, 2));\n\n revalidatePath(`/`);\n }\n\nAdding linked data\n==================\n\n.. edb:split-section::\n\n For the ``addCard`` action, you need to insert a new ``Card`` object and update the ``Deck.cards`` set to include the new ``Card`` object. Notice that the ``order`` property is set by selecting the maximum ``order`` property of this ``Deck.cards`` set and incrementing it by 1.\n\n The syntax for adding an object to a set of links is ``{ \"+=\": object }``. You can think of this as a shortcut for setting the link set to the current set plus the new object.\n\n .. code-block:: typescript-diff\n :caption: app/deck/[id]/actions.ts\n\n \"use server\";\n\n import { revalidatePath } from \"next/cache\";\n import { readFile, writeFile } from \"node:fs/promises\";\n import { client } from \"@/lib/gel\";\n import e from \"@/dbschema/edgeql-js\";\n import { Deck } from \"@/lib/models\";\n\n export async function updateDeck(formData: FormData) {\n const id = formData.get(\"id\");\n const name = formData.get(\"name\");\n const description = formData.get(\"description\");\n\n if (\n typeof id !== \"string\" ||\n (typeof name !== \"string\" &&\n typeof description !== \"string\")\n ) {\n return;\n }\n\n const nameSet = typeof name === \"string\" ? { name } : {};\n const descriptionSet =\n typeof description === \"string\" ? { description: description || null } : {};\n\n await e\n .update(e.Deck, (d) => ({\n filter_single: e.op(d.id, \"=\", e.uuid(id)),\n set: {\n ...nameSet,\n ...descriptionSet,\n },\n })).run(client);\n revalidatePath(`/deck/${id}`);\n }\n\n export async function addCard(formData: FormData) {\n const deckId = formData.get(\"deckId\");\n const front = formData.get(\"front\");\n const back = formData.get(\"back\");\n\n if (\n typeof deckId !== \"string\" ||\n typeof front !== \"string\" ||\n typeof back !== \"string\"\n ) {\n return;\n }\n\n - const decks = JSON.parse(await readFile(\"./decks.json\", \"utf-8\")) as Deck[];\n -\n - const deck = decks.find((deck) => deck.id === deckId);\n - if (!deck) {\n - return;\n - }\n -\n - deck.cards.push({ front, back, id: crypto.randomUUID() });\n - await writeFile(\"./decks.json\", JSON.stringify(decks, null, 2));\n + await e\n + .params(\n + {\n + front: e.str,\n + back: e.str,\n + deckId: e.uuid,\n + },\n + (params) => {\n + const deck = e.assert_exists(\n + e.select(e.Deck, (d) => ({\n + filter_single: e.op(d.id, \"=\", params.deckId),\n + }))\n + );\n +\n + const order = e.cast(e.int64, e.max(deck.cards.order));\n + const card = e.insert(e.Card, {\n + front: params.front,\n + back: params.back,\n + order: e.op(order, \"+\", 1),\n + });\n + return e.update(deck, (d) => ({\n + set: {\n + cards: {\n + \"+=\": card\n + },\n + },\n + }))\n + }\n + )\n + .run(client, {\n + front,\n + back,\n + deckId,\n + });\n\n revalidatePath(`/deck/${deckId}`);\n }\n\n export async function deleteCard(formData: FormData) {\n const cardId = formData.get(\"cardId\");\n\n if (typeof cardId !== \"string\") {\n return;\n }\n\n const decks = JSON.parse(await readFile(\"./decks.json\", \"utf-8\")) as Deck[];\n const deck = decks.find((deck) => deck.cards.some((card) => card.id === cardId));\n if (!deck) {\n return;\n }\n\n deck.cards = deck.cards.filter((card) => card.id !== cardId);\n await writeFile(\"./decks.json\", JSON.stringify(decks, null, 2));\n\n revalidatePath(`/`);\n }\n\nDeleting linked data\n====================\n\n.. edb:split-section::\n\n For the ``deleteCard`` action, delete the ``Card`` object and based on the deletion policy we set up earlier in the schema, the object will be deleted from the database and removed from the ``Deck.cards`` set.\n\n .. code-block:: typescript-diff\n :caption: app/deck/[id]/actions.ts\n\n \"use server\";\n\n import { revalidatePath } from \"next/cache\";\n - import { readFile, writeFile } from \"node:fs/promises\";\n import { client } from \"@/lib/gel\";\n import e from \"@/dbschema/edgeql-js\";\n import { Deck } from \"@/lib/models\";\n\n export async function updateDeck(formData: FormData) {\n const id = formData.get(\"id\");\n const name = formData.get(\"name\");\n const description = formData.get(\"description\");\n\n if (\n typeof id !== \"string\" ||\n (typeof name !== \"string\" &&\n typeof description !== \"string\")\n ) {\n return;\n }\n\n const nameSet = typeof name === \"string\" ? { name } : {};\n const descriptionSet =\n typeof description === \"string\" ? { description: description || null } : {};\n\n await e\n .update(e.Deck, (d) => ({\n filter_single: e.op(d.id, \"=\", e.uuid(id)),\n set: {\n ...nameSet,\n ...descriptionSet,\n },\n })).run(client);\n revalidatePath(`/deck/${id}`);\n }\n\n export async function addCard(formData: FormData) {\n const deckId = formData.get(\"deckId\");\n const front = formData.get(\"front\");\n const back = formData.get(\"back\");\n\n if (\n typeof deckId !== \"string\" ||\n typeof front !== \"string\" ||\n typeof back !== \"string\"\n ) {\n return;\n }\n\n await e\n .params(\n {\n front: e.str,\n back: e.str,\n deckId: e.uuid,\n },\n (params) => {\n const deck = e.assert_exists(\n e.select(e.Deck, (d) => ({\n filter_single: e.op(d.id, \"=\", params.deckId),\n }))\n );\n\n const order = e.cast(e.int64, e.max(deck.cards.order));\n const card = e.insert(e.Card, {\n front: params.front,\n back: params.back,\n order: e.op(order, \"+\", 1),\n });\n return e.update(deck, (d) => ({\n set: {\n cards: {\n \"+=\": card\n },\n },\n }))\n }\n )\n .run(client, {\n front,\n back,\n deckId,\n });\n\n revalidatePath(`/deck/${deckId}`);\n }\n\n export async function deleteCard(formData: FormData) {\n const cardId = formData.get(\"cardId\");\n\n if (typeof cardId !== \"string\") {\n return;\n }\n\n - const decks = JSON.parse(await readFile(\"./decks.json\", \"utf-8\")) as Deck[];\n - const deck = decks.find((deck) => deck.cards.some((card) => card.id === cardId));\n - if (!deck) {\n - return;\n - }\n -\n - deck.cards = deck.cards.filter((card) => card.id !== cardId);\n - await writeFile(\"./decks.json\", JSON.stringify(decks, null, 2));\n + await e\n + .params({ id: e.uuid }, (params) =>\n + e.delete(e.Card, (c) => ({\n + filter_single: e.op(c.id, \"=\", params.id),\n + }))\n + )\n + .run(client, { id: cardId });\n +\n\n revalidatePath(`/`);\n }\n\nQuerying data\n=============\n\n.. edb:split-section::\n\n Next, update the two ``queries.ts`` methods: ``getDecks`` and ``getDeck``.\n\n .. tabs::\n\n .. code-tab:: typescript-diff\n :caption: app/queries.ts\n\n - import { readFile } from \"node:fs/promises\";\n + import { client } from \"@/lib/gel\";\n + import e from \"@/dbschema/edgeql-js\";\n -\n - import { Deck } from \"@/lib/models\";\n\n export async function getDecks() {\n - const decks = JSON.parse(await readFile(\"./decks.json\", \"utf-8\")) as Deck[];\n + const decks = await e.select(e.Deck, (deck) => ({\n + id: true,\n + name: true,\n + description: true,\n + cards: e.select(deck.cards, (card) => ({\n + id: true,\n + front: true,\n + back: true,\n + order_by: card.order,\n + })),\n + })).run(client);\n\n return decks;\n }\n\n .. code-tab:: typescript-diff\n :caption: app/deck/[id]/queries.ts\n\n - import { readFile } from \"node:fs/promises\";\n - import { Deck } from \"@/lib/models\";\n + import { client } from \"@/lib/gel\";\n + import e from \"@/dbschema/edgeql-js\";\n\n export async function getDeck({ id }: { id: string }) {\n - const decks = JSON.parse(await readFile(\"./decks.json\", \"utf-8\")) as Deck[];\n - return decks.find((deck) => deck.id === id) ?? null;\n + return await e\n + .select(e.Deck, (deck) => ({\n + filter_single: e.op(deck.id, \"=\", e.uuid(id)),\n + id: true,\n + name: true,\n + description: true,\n + cards: e.select(deck.cards, (card) => ({\n + id: true,\n + front: true,\n + back: true,\n + order_by: card.order,\n + })),\n + }))\n + .run(client);\n }\n\n.. edb:split-section::\n\n In a terminal, run the Next.js development server.\n\n .. code-block:: sh\n\n $ npm run dev\n\n.. edb:split-section::\n\n A static JSON file to seed your database with a deck of trivia cards is included in the project. Open your browser and navigate to the app at ``_. Use the \"Import JSON\" button to import this JSON file into your database.\n\n .. image:: images/flashcards-import.png\n" }, { "path": "docs/intro/schema.rst", "content": ".. _ref_intro_schema:\n\n======\nSchema\n======\n\n\nThis page is intended as a rapid-fire overview of Gel's schema definition\nlanguage (SDL) so you can hit the ground running with Gel. Refer to the\nlinked pages for more in-depth documentation!\n\nScalar types\n------------\n\n|Gel| implements a rigorous type system containing the following primitive\ntypes.\n\n.. list-table::\n\n * - Strings\n - ``str``\n * - Booleans\n - ``bool``\n * - Numbers\n - ``int16`` ``int32`` ``int64`` ``float32`` ``float64``\n ``bigint`` ``decimal``\n * - UUID\n - ``uuid``\n * - JSON\n - ``json``\n * - Dates and times\n - ``datetime`` ``cal::local_datetime`` ``cal::local_date``\n ``cal::local_time``\n * - Durations\n - ``duration`` ``cal::relative_duration`` ``cal::date_duration``\n * - Binary data\n - ``bytes``\n * - Auto-incrementing counters\n - ``sequence``\n * - Enums\n - ``enum``\n\nThese primitives can be combined into arrays, tuples, and ranges.\n\n.. list-table::\n\n * - Arrays\n - ``array``\n * - Tuples (unnamed)\n - ``tuple``\n * - Tuples (named)\n - ``tuple``\n * - Ranges\n - ``range``\n\nCollectively, *primitive* and *collection* types comprise Gel's *scalar\ntype system*.\n\nObject types\n------------\n\nObject types are analogous to tables in SQL. They can contain **properties**,\nwhich can correspond to any scalar types, and **links**, which can correspond\nto any object types.\n\nProperties\n----------\n\nDeclare a property by naming it and setting its type.\n\n.. code-block:: sdl\n\n type Movie {\n title: str;\n }\n\nThe ``property`` keyword can be omitted for non-computed properties.\n\nSee :ref:`Schema > Object types `.\n\nRequired vs optional\n^^^^^^^^^^^^^^^^^^^^\n\nProperties are optional by default. Use the ``required`` keyword to make them\nrequired.\n\n.. code-block:: sdl\n\n type Movie {\n required title: str; # required\n release_year: int64; # optional\n }\n\nSee :ref:`Schema > Properties `.\n\nConstraints\n^^^^^^^^^^^\n\nAdd a pair of curly braces after the property to define additional\ninformation, including constraints.\n\n.. code-block:: sdl\n\n type Movie {\n required title: str {\n constraint exclusive;\n constraint min_len_value(8);\n constraint regexp(r'^[A-Za-z0-9 ]+$');\n }\n }\n\nSee :ref:`Schema > Constraints `.\n\n\nComputed properties\n^^^^^^^^^^^^^^^^^^^\n\nObject types can contain *computed properties* that correspond to EdgeQL\nexpressions. This expression is dynamically computed whenever the property is\nqueried.\n\n.. code-block:: sdl\n\n type Movie {\n required title: str;\n uppercase_title := str_upper(.title);\n }\n\nSee :ref:`Schema > Computeds `.\n\nLinks\n-----\n\nObject types can have links to other object types.\n\n.. code-block:: sdl\n\n type Movie {\n required title: str;\n director: Person;\n }\n\n type Person {\n required name: str;\n }\n\nThe ``link`` keyword can be omitted for non-computed links since Gel v3.\n\nUse the ``required`` and ``multi`` keywords to specify the cardinality of the\nrelation.\n\n.. code-block:: sdl\n\n type Movie {\n required title: str;\n\n cinematographer: Person; # zero or one\n required director: Person; # exactly one\n multi writers: Person; # zero or more\n required multi actors: Person; # one or more\n }\n\n type Person {\n required name: str;\n }\n\nTo define a one-to-one relation, use an ``exclusive`` constraint.\n\n.. code-block:: sdl\n\n type Movie {\n required title: str;\n required stats: MovieStats {\n constraint exclusive;\n };\n }\n\n type MovieStats {\n required budget: int64;\n required box_office: int64;\n }\n\nSee :ref:`Schema > Links `.\n\nComputed links\n^^^^^^^^^^^^^^\n\nObjects can contain \"computed links\": stored expressions that return a set of\nobjects. Computed links are dynamically computed when they are referenced in\nqueries. The example below defines a backlink.\n\n.. code-block:: sdl\n\n type Movie {\n required title: str;\n multi actors: Person;\n\n # returns all movies with same title\n multi same_title := (\n with t := .title\n select detached Movie filter .title = t\n )\n }\n\nBacklinks\n^^^^^^^^^\n\nA common use case for computed links is *backlinks*.\n\n.. code-block:: sdl\n\n type Movie {\n required title: str;\n multi actors: Person;\n }\n\n type Person {\n required name: str;\n multi acted_in := . Person``.\n\n``[is Movie]``\n This is a *type filter* that filters out all objects that aren't ``Movie``\n objects. A backlink still works without this filter, but could contain any\n other number of objects besides ``Movie`` objects.\n\nSee :ref:`Schema > Computeds > Backlinks `.\n\nConstraints\n-----------\n\nConstraints can also be defined at the *object level*.\n\n.. code-block:: sdl\n\n type BlogPost {\n title: str;\n author: User;\n\n constraint exclusive on ((.title, .author));\n }\n\nConstraints can contain exceptions; these are called *partial constraints*.\n\n.. code-block:: sdl\n\n type BlogPost {\n title: str;\n published: bool;\n\n constraint exclusive on (.title) except (not .published);\n }\n\nIndexes\n-------\n\nUse ``index on`` to define indexes on an object type.\n\n.. code-block:: sdl\n\n type Movie {\n required title: str;\n required release_year: int64;\n\n index on (.title); # simple index\n index on ((.title, .release_year)); # composite index\n index on (str_trim(str_lower(.title))); # computed index\n }\n\nThe ``id`` property, all links, and all properties with ``exclusive``\nconstraints are automatically indexed.\n\nSee :ref:`Schema > Indexes `.\n\nSchema mixins\n-------------\n\nObject types can be declared as ``abstract``. Non-abstract types can *extend*\nabstract types.\n\n.. code-block:: sdl\n\n abstract type Content {\n required title: str;\n }\n\n type Movie extending Content {\n required release_year: int64;\n }\n\n type TVShow extending Content {\n required num_seasons: int64;\n }\n\nMultiple inheritance is supported.\n\n.. code-block:: sdl\n\n abstract type HasTitle {\n required title: str;\n }\n\n abstract type HasReleaseYear {\n required release_year: int64;\n }\n\n type Movie extending HasTitle, HasReleaseYear {\n sequel_to: Movie;\n }\n\nSee :ref:`Schema > Object types > Inheritance\n`.\n\nPolymorphism\n------------\n\nLinks can correspond to abstract types. These are known as *polymorphic links*.\n\n.. code-block:: sdl\n\n abstract type Content {\n required title: str;\n }\n\n type Movie extending Content {\n required release_year: int64;\n }\n\n type TVShow extending Content {\n required num_seasons: int64;\n }\n\n type Franchise {\n required name: str;\n multi entries: Content;\n }\n\nSee :ref:`Schema > Links > Polymorphism\n` and :ref:`EdgeQL > Select > Polymorphic\nqueries `.\n\n" }, { "path": "docs/intro/tutorials/ai_fastapi_searchbot.rst", "content": ".. _ref_guide_fastapi_gelai_searchbot:\n\n===============================\nBuild a Search Bot with FastAPI\n===============================\n\n:edb-alt-title: Building a search bot with memory using FastAPI and Gel AI\n\nIn this tutorial we're going to walk you through building a chat bot with search\ncapabilities using Gel and `FastAPI `_.\n\nFastAPI is a framework designed to help you build web apps *fast*. Gel is a\ndata layer designed to help you figure out storage in your application - also\n*fast*. By the end of this tutorial, you will have tried out different aspects\nof using those two together.\n\nWe will start by creating an app with FastAPI, adding web search capabilities,\nand then putting search results through a language model to get a\nhuman-friendly answer. After that, we'll use Gel to implement chat history so\nthat the bot remembers previous interactions with the user. We'll finish it off\nwith semantic search-based cross-chat memory.\n\n\n1. Initialize the project\n=========================\n\n.. edb:split-section::\n\n We're going to start by installing `uv `_ - a Python\n package manager that's going to simplify environment management for us. You can\n follow their `installation instructions\n `_ or simply run:\n\n .. code-block:: bash\n\n $ curl -LsSf https://astral.sh/uv/install.sh | sh\n\n.. edb:split-section::\n\n Once that is done, we can use uv to create scaffolding for our project following\n the `documentation `_:\n\n .. code-block:: bash\n\n $ uv init searchbot \\\n && cd searchbot\n\n.. edb:split-section::\n\n For now, we know we're going to need Gel and FastAPI, so let's add those\n following uv's instructions on `managing dependencies\n `_,\n as well as FastAPI's `installation docs\n `_. Running ``uv sync`` after\n that will create our virtual environment in a ``.venv`` directory and ensure\n it's ready. As the last step, we'll activate the environment and get started.\n\n .. note::\n\n Every time you open a new terminal session, you should source the\n environment before running ``python``, ``gel`` or ``fastapi`` commands.\n\n .. code-block:: bash\n\n $ uv add \"fastapi[standard]\" \\\n && uv add gel \\\n && uv sync \\\n && source .venv/bin/activate\n\n\n2. Get started with FastAPI\n===========================\n\n.. edb:split-section::\n\n At this stage we need to follow FastAPI's `tutorial\n `_ to create the foundation of our app.\n\n We're going to make a minimal web API with one endpoint that takes in a user\n query as an input and echoes it as an output. First, let's make a directory\n called ``app`` in our project root, and put an empty ``__init__.py`` there.\n\n .. code-block:: bash\n\n $ mkdir app && touch app/__init__.py\n\n.. edb:split-section::\n\n Now let's create a file called ``main.py`` inside the ``app`` directory and put\n the \"Hello World\" example in it:\n\n .. code-block:: python\n :caption: app/main.py\n\n from fastapi import FastAPI\n\n app = FastAPI()\n\n\n @app.get(\"/\")\n async def root():\n return {\"message\": \"Hello World\"}\n\n\n.. edb:split-section::\n\n To start the server, we'll run:\n\n .. code-block:: bash\n\n $ fastapi dev app/main.py\n\n\n.. edb:split-section::\n\n Once the server gets up and running, we can make sure it works using FastAPI's\n built-in UI at _, or manually with ``curl``:\n\n .. code-block:: bash\n\n $ curl -X 'GET' \\\n 'http://127.0.0.1:8000/' \\\n -H 'accept: application/json'\n\n {\"message\":\"Hello World\"}\n\n\n.. edb:split-section::\n\n Now, to create the search endpoint we mentioned earlier, we need to pass our\n query as a parameter to it. We'd prefer to have it in the request's body\n since user messages can be long.\n\n In FastAPI land, this is done by creating a Pydantic schema and making it the\n type of the input parameter. `Pydantic `_ is\n a data validation library for Python. It has many features, but we don't\n actually need to know about them for now. All we need to know is that FastAPI\n uses Pydantic types to automatically figure out schemas for `input\n `_, as well as `output\n `_.\n\n Let's add the following to our ``main.py``:\n\n .. code-block:: python\n :caption: app/main.py\n\n from pydantic import BaseModel\n\n\n class SearchTerms(BaseModel):\n query: str\n\n class SearchResult(BaseModel):\n response: str | None = None\n\n\n.. edb:split-section::\n\n Now, we can define our endpoint. We'll set the two classes we just created as\n the new endpoint's argument and return type.\n\n .. code-block:: python\n :caption: app/main.py\n\n @app.post(\"/search\")\n async def search(search_terms: SearchTerms) -> SearchResult:\n return SearchResult(response=search_terms.query)\n\n\n.. edb:split-section::\n\n Same as before, we can test the endpoint using the UI, or by sending a request\n with ``curl``:\n\n .. code-block:: bash\n\n $ curl -X 'POST' \\\n 'http://127.0.0.1:8000/search' \\\n -H 'accept: application/json' \\\n -H 'Content-Type: application/json' \\\n -d '{ \"query\": \"string\" }'\n\n {\n \"response\": \"string\",\n }\n\n3. Implement web search\n=======================\n\nNow that we have our web app infrastructure in place, let's add some substance\nto it by implementing web search capabilities.\n\n.. edb:split-section::\n\n There're many powerful feature-rich products for LLM-driven web search. But\n in this tutorial we're going to use a much more reliable source of real-world\n information that is comment threads on `Hacker News\n `_. Their `web API\n `_ is free of charge and doesn't require an\n account. Below is a simple function that requests a full-text search for a\n string query and extracts a nice sampling of comment threads from each of the\n stories that came up in the result.\n\n We are not going to cover this code sample in too much depth. Feel free to grab\n it save it to ``app/web.py``, or make your own.\n\n Notice that we've created another Pydantic type called ``WebSource`` to store\n our web search results. There's no framework-related reason for that, it's just\n nicer than passing dictionaries around.\n\n .. code-block:: python\n :caption: app/web.py\n :class: collapsible\n\n import requests\n from pydantic import BaseModel\n from datetime import datetime\n import html\n\n\n class WebSource(BaseModel):\n \"\"\"Type that stores search results.\"\"\"\n\n url: str | None = None\n title: str | None = None\n text: str | None = None\n\n\n def extract_comment_thread(\n comment: dict,\n max_depth: int = 3,\n current_depth: int = 0,\n max_children=3,\n ) -> list[str]:\n \"\"\"\n Recursively extract comments from a thread up to max_depth.\n Returns a list of formatted comment strings.\n \"\"\"\n if not comment or current_depth > max_depth:\n return []\n\n results = []\n\n # Get timestamp, author and the body of the comment,\n # then pad it with spaces so that it's offset appropriately for its depth\n\n if comment[\"text\"]:\n timestamp = datetime.fromisoformat(comment[\"created_at\"].replace(\"Z\", \"+00:00\"))\n author = comment[\"author\"]\n text = html.unescape(comment[\"text\"])\n formatted_comment = f\"[{timestamp.strftime('%Y-%m-%d %H:%M')}] {author}: {text}\"\n results.append((\" \" * current_depth) + formatted_comment)\n\n # If there're children comments, we are going to extract them too,\n # and add them to the list.\n\n if comment.get(\"children\"):\n for child in comment[\"children\"][:max_children]:\n child_comments = extract_comment_thread(child, max_depth, current_depth + 1)\n results.extend(child_comments)\n\n return results\n\n\n def fetch_web_sources(query: str, limit: int = 5) -> list[WebSource]:\n \"\"\"\n For a given query perform a full-text search for stories on Hacker News.\n From each of the matched stories extract the comment thread and format it into a single string.\n For each story return its title, url and comment thread.\n \"\"\"\n search_url = \"http://hn.algolia.com/api/v1/search_by_date?numericFilters=num_comments>0\"\n\n # Search for stories\n response = requests.get(\n search_url,\n params={\n \"query\": query,\n \"tags\": \"story\",\n \"hitsPerPage\": limit,\n \"page\": 0,\n },\n )\n\n response.raise_for_status()\n search_result = response.json()\n\n # For each search hit fetch and process the story\n web_sources = []\n for hit in search_result.get(\"hits\", []):\n item_url = f\"https://hn.algolia.com/api/v1/items/{hit['story_id']}\"\n response = requests.get(item_url)\n response.raise_for_status()\n item_result = response.json()\n\n site_url = f\"https://news.ycombinator.com/item?id={hit['story_id']}\"\n title = hit[\"title\"]\n comments = extract_comment_thread(item_result)\n text = \"\\n\".join(comments) if len(comments) > 0 else None\n web_sources.append(\n WebSource(url=site_url, title=title, text=text)\n )\n\n return web_sources\n\n\n if __name__ == \"__main__\":\n web_sources = fetch_web_sources(\"edgedb\", limit=5)\n\n for source in web_sources:\n print(source.url)\n print(source.title)\n print(source.text)\n\n\n.. edb:split-section::\n\n One more note: this snippet comes with an extra dependency called ``requests``,\n which is a library for making HTTP requests. Let's add it by running:\n\n .. code-block:: bash\n\n $ uv add requests\n\n\n.. edb:split-section::\n\n Now, we can test our web search on its own by running it like this:\n\n .. code-block:: bash\n\n $ python3 app/web.py\n\n\n.. edb:split-section::\n\n It's time to reflect the new capabilities in our web app.\n\n .. code-block:: python\n :caption: app/main.py\n\n from .web import fetch_web_sources, WebSource\n\n async def search_web(query: str) -> list[WebSource]:\n raw_sources = fetch_web_sources(query, limit=5)\n return [s for s in raw_sources if s.text is not None]\n\n\n.. edb:split-section::\n\n Now we can update the ``/search`` endpoint as follows:\n\n .. code-block:: python-diff\n :caption: app/main.py\n\n class SearchResult(BaseModel):\n response: str | None = None\n + sources: list[WebSource] | None = None\n\n\n @app.post(\"/search\")\n async def search(search_terms: SearchTerms) -> SearchResult:\n + web_sources = await search_web(search_terms.query)\n - return SearchResult(response=search_terms.query)\n + return SearchResult(\n + response=search_terms.query, sources=web_sources\n + )\n\n\n4. Connect to the LLM\n=====================\n\nNow that we're capable of scraping text from search results, we can forward\nthose results to the LLM to get a nice-looking summary.\n\n.. edb:split-section::\n\n There's a million different LLMs accessible via a web API (`one\n `_, `two\n `_, `three\n `_, `four `_ to name\n a few), feel free to choose whichever you prefer. In this tutorial we will\n roll with OpenAI, primarily for how ubiquitous it is. To keep things somewhat\n provider-agnostic, we're going to get completions via raw HTTP requests.\n Let's grab API descriptions from OpenAI's `API documentation\n `_, and set up\n LLM generation like this:\n\n .. code-block:: python\n :caption: app/main.py\n\n import requests\n from dotenv import load_dotenv\n\n _ = load_dotenv()\n\n\n def get_llm_completion(system_prompt: str, messages: list[dict[str, str]]) -> str:\n api_key = os.getenv(\"OPENAI_API_KEY\")\n url = \"https://api.openai.com/v1/chat/completions\"\n headers = {\"Content-Type\": \"application/json\", \"Authorization\": f\"Bearer {api_key}\"}\n\n response = requests.post(\n url,\n headers=headers,\n json={\n \"model\": \"gpt-4o-mini\",\n \"messages\": [\n {\"role\": \"developer\", \"content\": system_prompt},\n *messages,\n ],\n },\n )\n response.raise_for_status()\n result = response.json()\n return result[\"choices\"][0][\"message\"][\"content\"]\n\n\n.. edb:split-section::\n\n Note that this cloud LLM API (and many others) requires a secret key to be\n set as an environment variable. A common way to manage those is to use the\n ``python-dotenv`` library in combinations with a ``.env`` file. Feel free to\n browse `the readme\n `_,\n to learn more. Create a file called ``.env`` in the root directory and put\n your api key in there:\n\n .. code-block:: .env\n :caption: .env\n\n OPENAI_API_KEY=\"sk-...\"\n\n\n.. edb:split-section::\n\n Don't forget to add the new dependency to the environment:\n\n .. code-block:: bash\n\n uv add python-dotenv\n\n\n.. edb:split-section::\n\n And now we can integrate this LLM-related code with the rest of the app. First,\n let's set up a function that prepares LLM inputs:\n\n\n .. code-block:: python\n :caption: app/main.py\n\n async def generate_answer(\n query: str,\n web_sources: list[WebSource],\n ) -> SearchResult:\n system_prompt = (\n \"You are a helpful assistant that answers user's questions\"\n + \" by finding relevant information in Hacker News threads.\"\n + \" When answering the question, describe conversations that people have around the subject,\"\n + \" provided to you as a context, or say i don't know if they are completely irrelevant.\"\n )\n\n prompt = f\"User search query: {query}\\n\\nWeb search results:\\n\"\n\n for i, source in enumerate(web_sources):\n prompt += f\"Result {i} (URL: {source.url}):\\n\"\n prompt += f\"{source.text}\\n\\n\"\n\n messages = [{\"role\": \"user\", \"content\": prompt}]\n\n llm_response = get_llm_completion(\n system_prompt=system_prompt,\n messages=messages,\n )\n\n search_result = SearchResult(\n response=llm_response,\n sources=web_sources,\n )\n\n return search_result\n\n\n.. edb:split-section::\n\n Then we can plug that function into the ``/search`` endpoint:\n\n .. code-block:: python-diff\n :caption: app/main.py\n\n @app.post(\"/search\")\n async def search(search_terms: SearchTerms) -> SearchResult:\n web_sources = await search_web(search_terms.query)\n + search_result = await generate_answer(search_terms.query, web_sources)\n + return search_result\n - return SearchResult(\n - response=search_terms.query, sources=web_sources\n - )\n\n\n.. edb:split-section::\n\n And now we can test the result as usual.\n\n .. code-block:: bash\n\n $ curl -X 'POST' \\\n 'http://127.0.0.1:8000/search' \\\n -H 'accept: application/json' \\\n -H 'Content-Type: application/json' \\\n -d '{ \"query\": \"gel\" }'\n\n\n5. Use Gel to implement chat history\n====================================\n\nSo far we've built an application that can take in a query, fetch some Hacker\nNews threads for it, sift through them using an LLM, and generate a nice\nsummary.\n\nHowever, right now it's hardly user-friendly since you have to speak in\nkeywords and basically start over every time you want to refine the query. To\nenable a more organic multi-turn interaction, we need to add chat history and\ninfer the query from the context of the entire conversation.\n\nNow's a good time to introduce Gel.\n\n.. edb:split-section::\n\n In case you need installation instructions, take a look at the :ref:`Quickstart\n `. Once Gel CLI is present in your system, initialize the\n project like this:\n\n .. code-block:: bash\n\n $ gel project init --non-interactive\n\n\nThis command is going to put some project scaffolding inside our app, spin up a\nlocal instace of Gel, and then link the two together. From now on, all\nGel-related things that happen inside our project directory are going to be\nautomatically run on the correct database instance, no need to worry about\nconnection incantations.\n\n\nDefining the schema\n-------------------\n\nThe database :ref:`schema ` in Gel is defined\ndeclaratively. The :gelcmd:`project init` command has created a file called\n:dotgel:`dbschema/default`, which we're going to use to define our types.\n\n.. edb:split-section::\n\n We obviously want to keep track of the messages, so we need to represent\n those in the schema. By convention established in the LLM space, each message\n is going to have a role in addition to the message content itself. We can\n also get Gel to automatically keep track of message's creation time by adding\n a property callled ``timestamp`` and setting its :ref:`default value\n ` to the output of the :ref:`datetime_current()\n ` function. Finally, LLM messages in our search bot have\n source URLs associated with them. Let's keep track of those too, by adding a\n :ref:`multi-property `.\n\n .. code-block:: sdl\n :caption: dbschema/default.esdl\n\n type Message {\n role: str;\n body: str;\n timestamp: datetime {\n default := datetime_current();\n }\n multi sources: str;\n }\n\n\n.. edb:split-section::\n\n Messages are grouped together into a chat, so let's add that entity to our\n schema too.\n\n .. code-block:: sdl\n :caption: dbschema/default.esdl\n\n type Chat {\n multi messages: Message;\n }\n\n\n.. edb:split-section::\n\n And chats all belong to a certain user, making up their chat history. One other\n thing we'd like to keep track of about our users is their username, and it would\n make sense for us to make sure that it's unique by using an ``excusive``\n :ref:`constraint `.\n\n .. code-block:: sdl\n :caption: dbschema/default.esdl\n\n type User {\n name: str {\n constraint exclusive;\n }\n multi chats: Chat;\n }\n\n\n.. edb:split-section::\n\n We're going to keep our schema super simple. One cool thing about Gel is that\n it will enable us to easily implement advanced features such as authentication\n or AI down the road, but we're gonna come back to that later.\n\n For now, this is the entire schema we came up with:\n\n .. code-block:: sdl\n :caption: dbschema/default.esdl\n\n module default {\n type Message {\n role: str;\n body: str;\n timestamp: datetime {\n default := datetime_current();\n }\n multi sources: str;\n }\n\n type Chat {\n multi messages: Message;\n }\n\n type User {\n name: str {\n constraint exclusive;\n }\n multi chats: Chat;\n }\n }\n\n\n.. edb:split-section::\n\n Let's use the :gelcmd:`migration create` CLI command, followed by :gelcmd:`migrate` in\n order to migrate to our new schema and proceed to writing some queries.\n\n .. code-block:: bash\n\n $ gel migration create\n $ gel migrate\n\n\n.. edb:split-section::\n\n Now that our schema is applied, let's quickly populate the database with some\n fake data in order to be able to test the queries. We're going to explore\n writing queries in a bit, but for now you can just run the following command in\n the shell:\n\n .. code-block:: bash\n :class: collapsible\n\n $ mkdir app/sample_data && cat << 'EOF' > app/sample_data/inserts.edgeql\n # Create users first\n insert User {\n name := 'alice',\n };\n insert User {\n name := 'bob',\n };\n # Insert chat histories for Alice\n update User\n filter .name = 'alice'\n set {\n chats := {\n (insert Chat {\n messages := {\n (insert Message {\n role := 'user',\n body := 'What are the main differences between GPT-3 and GPT-4?',\n timestamp := '2024-01-07T10:00:00Z',\n sources := {'arxiv:2303.08774', 'openai.com/research/gpt-4'}\n }),\n (insert Message {\n role := 'assistant',\n body := 'The key differences include improved reasoning capabilities, better context understanding, and enhanced safety features...',\n timestamp := '2024-01-07T10:00:05Z',\n sources := {'openai.com/blog/gpt-4-details', 'arxiv:2303.08774'}\n })\n }\n }),\n (insert Chat {\n messages := {\n (insert Message {\n role := 'user',\n body := 'Can you explain what policy gradient methods are in RL?',\n timestamp := '2024-01-08T14:30:00Z',\n sources := {'Sutton-Barto-RL-Book-Ch13', 'arxiv:1904.12901'}\n }),\n (insert Message {\n role := 'assistant',\n body := 'Policy gradient methods are a class of reinforcement learning algorithms that directly optimize the policy...',\n timestamp := '2024-01-08T14:30:10Z',\n sources := {'Sutton-Barto-RL-Book-Ch13', 'spinning-up.openai.com'}\n })\n }\n })\n }\n };\n # Insert chat histories for Bob\n update User\n filter .name = 'bob'\n set {\n chats := {\n (insert Chat {\n messages := {\n (insert Message {\n role := 'user',\n body := 'What are the pros and cons of different sharding strategies?',\n timestamp := '2024-01-05T16:15:00Z',\n sources := {'martin-kleppmann-ddia-ch6', 'aws.amazon.com/sharding-patterns'}\n }),\n (insert Message {\n role := 'assistant',\n body := 'The main sharding strategies include range-based, hash-based, and directory-based sharding...',\n timestamp := '2024-01-05T16:15:08Z',\n sources := {'martin-kleppmann-ddia-ch6', 'mongodb.com/docs/sharding'}\n }),\n (insert Message {\n role := 'user',\n body := 'Could you elaborate on hash-based sharding?',\n timestamp := '2024-01-05T16:16:00Z',\n sources := {'mongodb.com/docs/sharding'}\n })\n }\n })\n }\n };\n EOF\n\n\n.. edb:split-section::\n\n This created the ``app/sample_data/inserts.edgeql`` file, which we can now execute\n using the CLI like this:\n\n .. code-block:: bash\n\n $ gel query -f app/sample_data/inserts.edgeql\n\n {\"id\": \"862de904-de39-11ef-9713-4fab09220c4a\"}\n {\"id\": \"862e400c-de39-11ef-9713-2f81f2b67013\"}\n {\"id\": \"862de904-de39-11ef-9713-4fab09220c4a\"}\n {\"id\": \"862e400c-de39-11ef-9713-2f81f2b67013\"}\n\n\n.. edb:split-section::\n\n The :gelcmd:`query` command is one of many ways we can execute a query in Gel. Now\n that we've done it, there's stuff in the database.\n\n Let's verify it by running:\n\n .. code-block:: bash\n\n $ gel query \"select User { name };\"\n\n {\"name\": \"alice\"}\n {\"name\": \"bob\"}\n\n\nWriting queries\n---------------\n\nWith schema in place, it's time to focus on getting the data in and out of the\ndatabase.\n\nIn this tutorial we're going to write queries using :ref:`EdgeQL\n` and then use :ref:`codegen ` to\ngenerate typesafe function that we can plug directly into out Python code. If\nyou are completely unfamiliar with EdgeQL, now is a good time to check out the\nbasics before proceeding.\n\n\n.. edb:split-section::\n\n Let's move on. First, we'll create a directory inside ``app`` called\n ``queries``. This is where we're going to put all of the EdgeQL-related stuff.\n\n We're going to start by writing a query that fetches all of the users. In\n ``queries`` create a file named ``get_users.edgeql`` and put the following query\n in there:\n\n .. code-block:: edgeql\n :caption: app/queries/get_users.edgeql\n\n select User { name };\n\n\n.. edb:split-section::\n\n Now run the code generator from the shell:\n\n .. code-block:: bash\n\n $ gel-py\n\n\n.. edb:split-section::\n\n It's going to automatically locate the ``.edgeql`` file and generate types for\n it. We can inspect generated code in ``app.queries/get_users_async_edgeql.py``.\n Once that is done, let's use those types to create the endpoint in ``main.py``:\n\n .. code-block:: python\n :caption: app/main.py\n\n from edgedb import create_async_client\n from .queries.get_users_async_edgeql import get_users as get_users_query, GetUsersResult\n\n\n gel_client = create_async_client()\n\n @app.get(\"/users\")\n async def get_users() -> list[GetUsersResult]:\n return await get_users_query(gel_client)\n\n\n.. edb:split-section::\n\n Let's verify it that works as expected:\n\n .. code-block:: bash\n\n $ curl -X 'GET' \\\n 'http://127.0.0.1:8000/users' \\\n -H 'accept: application/json'\n\n [\n {\n \"id\": \"862de904-de39-11ef-9713-4fab09220c4a\",\n \"name\": \"alice\"\n },\n {\n \"id\": \"862e400c-de39-11ef-9713-2f81f2b67013\",\n \"name\": \"bob\"\n }\n ]\n\n\n.. edb:split-section::\n\n While we're at it, let's also implement the option to fetch a user by their\n username. In order to do that, we need to write a new query in a separate file\n ``app/queries/get_user_by_name.edgeql``:\n\n .. code-block:: edgeql\n :caption: app/queries/get_user_by_name.edgeql\n\n select User { name }\n filter .name = $name;\n\n\n.. edb:split-section::\n\n After that, we will run the code generator again by calling ``gel-py``. In the\n app, we are going to reuse the same endpoint that fetches the list of all users.\n From now on, if the user calls it without any arguments (e.g.\n ``http://127.0.0.1/users``), they are going to receive the list of all users,\n same as before. But if they pass a username as a query argument like this:\n ``http://127.0.0.1/users?username=bob``, the system will attempt to fetch a user\n named ``bob``.\n\n In order to achieve this, we're going to need to add a ``Query``-type argument\n to our endpoint function. You can learn more about how to configure this type of\n arguments in `FastAPI's docs\n `_. It's default value is\n going to be ``None``, which will enable us to implement our conditional logic:\n\n .. code-block:: python\n :caption: app/main.py\n\n from fastapi import Query, HTTPException\n from http import HTTPStatus\n from .queries.get_user_by_name_async_edgeql import (\n get_user_by_name as get_user_by_name_query,\n GetUserByNameResult,\n )\n\n\n @app.get(\"/users\")\n async def get_users(\n username: str = Query(None),\n ) -> list[GetUsersResult] | GetUserByNameResult:\n \"\"\"List all users or get a user by their username\"\"\"\n if username:\n user = await get_user_by_name_query(gel_client, name=username)\n if not user:\n raise HTTPException(\n HTTPStatus.NOT_FOUND,\n detail={\"error\": f\"Error: user {username} does not exist.\"},\n )\n return user\n else:\n return await get_users_query(gel_client)\n\n\n.. edb:split-section::\n\n And once again, let's verify that everything works:\n\n .. code-block:: bash\n\n $ curl -X 'GET' \\\n 'http://127.0.0.1:8000/users?username=alice' \\\n -H 'accept: application/json'\n\n {\n \"id\": \"862de904-de39-11ef-9713-4fab09220c4a\",\n \"name\": \"alice\"\n }\n\n\n.. edb:split-section::\n\n Finally, let's also implement the option to add a new user. For this, just as\n before, we'll create a new file ``app/queries/create_user.edgeql``, add a query\n to it and run code generation.\n\n Note that in this query we've wrapped the ``insert`` in a ``select`` statement.\n This is a common pattern in EdgeQL, that can be used whenever you would like to\n get something other than object ID when you just inserted it.\n\n .. code-block:: edgeql\n :caption: app/queries/create_user.edgeql\n\n select(\n insert User {\n name := $username\n }\n ) {\n name\n }\n\n\n\n.. edb:split-section::\n\n In order to integrate this query into our app, we're going to add a new\n endpoint. Note that this one has the same name ``/users``, but is for the POST\n HTTP method.\n\n .. code-block:: python\n :caption: app/main.py\n\n from gel import ConstraintViolationError\n from .queries.create_user_async_edgeql import (\n create_user as create_user_query,\n CreateUserResult,\n )\n\n @app.post(\"/users\", status_code=HTTPStatus.CREATED)\n async def post_user(username: str = Query()) -> CreateUserResult:\n try:\n return await create_user_query(gel_client, username=username)\n except ConstraintViolationError:\n raise HTTPException(\n status_code=HTTPStatus.BAD_REQUEST,\n detail={\"error\": f\"Username '{username}' already exists.\"},\n )\n\n\n.. edb:split-section::\n\n Once more, let's verify that the new endpoint works as expected:\n\n .. code-block:: bash\n\n $ curl -X 'POST' \\\n 'http://127.0.0.1:8000/users?username=charlie' \\\n -H 'accept: application/json' \\\n -d ''\n\n {\n \"id\": \"20372a1a-ded5-11ef-9a08-b329b578c45c\",\n \"name\": \"charlie\"\n }\n\n\n.. edb:split-section::\n\n This wraps things up for our user-related functionality. Of course, we now need\n to deal with Chats and Messages, too. We're not going to go in depth for those,\n since the process would be quite similar to what we've just done. Instead, feel\n free to implement those endpoints yourself as an exercise, or copy the code\n below if you are in rush.\n\n .. code-block:: bash\n :class: collapsible\n\n $ echo 'select Chat {\n messages: { role, body, sources },\n user := .$username;' > app/queries/get_chats.edgeql && echo 'select Chat {\n messages: { role, body, sources },\n user := .$username and .id = $chat_id;' > app/queries/get_chat_by_id.edgeql && echo 'with new_chat := (insert Chat)\n select (\n update User filter .name = $username\n set {\n chats := assert_distinct(.chats union new_chat)\n }\n ) {\n new_chat_id := new_chat.id\n }' > app/queries/create_chat.edgeql && echo 'with\n user := (select User filter .name = $username),\n chat := (\n select Chat filter .$chat_id\n )\n select Message {\n role,\n body,\n sources,\n chat := . app/queries/get_messages.edgeql && echo 'with\n user := (select User filter .name = $username),\n update Chat\n filter .id = $chat_id and .$message_role,\n body := $message_body,\n sources := array_unpack(>$sources)\n }\n ))\n }' > app/queries/add_message.edgeql\n\n\n.. edb:split-section::\n\n And these are the endpoint definitions, provided in bulk.\n\n .. code-block:: python\n :caption: app/main.py\n :class: collapsible\n\n from .queries.get_chats_async_edgeql import get_chats as get_chats_query, GetChatsResult\n from .queries.get_chat_by_id_async_edgeql import (\n get_chat_by_id as get_chat_by_id_query,\n GetChatByIdResult,\n )\n from .queries.get_messages_async_edgeql import (\n get_messages as get_messages_query,\n GetMessagesResult,\n )\n from .queries.create_chat_async_edgeql import (\n create_chat as create_chat_query,\n CreateChatResult,\n )\n from .queries.add_message_async_edgeql import (\n add_message as add_message_query,\n )\n\n\n @app.get(\"/chats\")\n async def get_chats(\n username: str = Query(), chat_id: str = Query(None)\n ) -> list[GetChatsResult] | GetChatByIdResult:\n \"\"\"List user's chats or get a chat by username and id\"\"\"\n if chat_id:\n chat = await get_chat_by_id_query(\n gel_client, username=username, chat_id=chat_id\n )\n if not chat:\n raise HTTPException(\n HTTPStatus.NOT_FOUND,\n detail={\"error\": f\"Chat {chat_id} for user {username} does not exist.\"},\n )\n return chat\n else:\n return await get_chats_query(gel_client, username=username)\n\n\n @app.post(\"/chats\", status_code=HTTPStatus.CREATED)\n async def post_chat(username: str) -> CreateChatResult:\n return await create_chat_query(gel_client, username=username)\n\n\n @app.get(\"/messages\")\n async def get_messages(\n username: str = Query(), chat_id: str = Query()\n ) -> list[GetMessagesResult]:\n \"\"\"Fetch all messages from a chat\"\"\"\n return await get_messages_query(gel_client, username=username, chat_id=chat_id)\n\n\n.. edb:split-section::\n\n For the ``post_messages`` function we're going to do something a little bit\n different though. Since this is now the primary way for the user to add their\n queries to the system, it functionally superceeds the ``/search`` endpoint we\n made before. To this end, this function is where we're going to handle saving\n messages, retrieving chat history, invoking web search and generating the\n answer.\n\n .. code-block:: python-diff\n :caption: app/main.py\n\n - @app.post(\"/search\")\n - async def search(search_terms: SearchTerms) -> SearchResult:\n - web_sources = await search_web(search_terms.query)\n - search_result = await generate_answer(search_terms.query, web_sources)\n - return search_result\n\n + @app.post(\"/messages\", status_code=HTTPStatus.CREATED)\n + async def post_messages(\n + search_terms: SearchTerms,\n + username: str = Query(),\n + chat_id: str = Query(),\n + ) -> SearchResult:\n + chat_history = await get_messages_query(\n + gel_client, username=username, chat_id=chat_id\n + )\n\n + _ = await add_message_query(\n + gel_client,\n + username=username,\n + message_role=\"user\",\n + message_body=search_terms.query,\n + sources=[],\n + chat_id=chat_id,\n + )\n\n + search_query = search_terms.query\n + web_sources = await search_web(search_query)\n\n + search_result = await generate_answer(\n + search_terms.query, chat_history, web_sources\n + )\n\n + _ = await add_message_query(\n + gel_client,\n + username=username,\n + message_role=\"assistant\",\n + message_body=search_result.response,\n + sources=search_result.sources,\n + chat_id=chat_id,\n + )\n\n + return search_result\n\n\n.. edb:split-section::\n\n Let's not forget to modify the ``generate_answer`` function, so it can also be\n history-aware.\n\n .. code-block:: python-diff\n :caption: app/main.py\n\n async def generate_answer(\n query: str,\n + chat_history: list[GetMessagesResult],\n web_sources: list[WebSource],\n ) -> SearchResult:\n system_prompt = (\n \"You are a helpful assistant that answers user's questions\"\n + \" by finding relevant information in HackerNews threads.\"\n + \" When answering the question, describe conversations that people have around the subject,\"\n + \" provided to you as a context, or say i don't know if they are completely irrelevant.\"\n )\n\n prompt = f\"User search query: {query}\\n\\nWeb search results:\\n\"\n\n for i, source in enumerate(web_sources):\n prompt += f\"Result {i} (URL: {source.url}):\\n\"\n prompt += f\"{source.text}\\n\\n\"\n\n - messages = [{\"role\": \"user\", \"content\": prompt}]\n + messages = [\n + {\"role\": message.role, \"content\": message.body} for message in chat_history\n + ]\n + messages.append({\"role\": \"user\", \"content\": prompt})\n\n llm_response = get_llm_completion(\n system_prompt=system_prompt,\n messages=messages,\n )\n\n search_result = SearchResult(\n response=llm_response,\n sources=web_sources,\n )\n\n return search_result\n\n\n.. edb:split-section::\n\n Ok, this should be it for setting up the chat history. Let's test it. First, we\n are going to start a new chat for our user:\n\n .. code-block:: bash\n\n $ curl -X 'POST' \\\n 'http://127.0.0.1:8000/chats?username=charlie' \\\n -H 'accept: application/json' \\\n -d ''\n\n {\n \"id\": \"20372a1a-ded5-11ef-9a08-b329b578c45c\",\n \"new_chat_id\": \"544ef3f2-ded8-11ef-ba16-f7f254b95e36\"\n }\n\n\n.. edb:split-section::\n\n Next, let's add a couple messages and wait for the bot to respond:\n\n .. code-block:: bash\n\n $ curl -X 'POST' \\\n 'http://127.0.0.1:8000/messages?username=charlie&chat_id=544ef3f2-ded8-11ef-ba16-f7f254b95e36' \\\n -H 'accept: application/json' \\\n -H 'Content-Type: application/json' \\\n -d '{\n \"query\": \"best database in existence\"\n }'\n\n $ curl -X 'POST' \\\n 'http://127.0.0.1:8000/messages?username=charlie&chat_id=544ef3f2-ded8-11ef-ba16-f7f254b95e36' \\\n -H 'accept: application/json' \\\n -H 'Content-Type: application/json' \\\n -d '{\n \"query\": \"gel\"\n }'\n\n\n.. edb:split-section::\n\n Finally, let's check that the messages we saw are in fact stored in the chat\n history:\n\n .. code-block:: bash\n\n $ curl -X 'GET' \\\n 'http://127.0.0.1:8000/messages?username=charlie&chat_id=544ef3f2-ded8-11ef-ba16-f7f254b95e36' \\\n -H 'accept: application/json'\n\n\nIn reality this workflow would've been handled by the frontend, providing the\nuser with a nice inteface to interact with. But even without one our chatbot is\nalmost functional by now.\n\nGenerating a Google search query\n--------------------------------\n\nCongratulations! We just got done implementing multi-turn conversations for our\nsearch bot.\n\nHowever, there's still one crucial piece missing. Right now we're simply\nforwarding the users message straight to the full-text search. But what happens\nif their message is a followup that cannot be used as a standalone search\nquery?\n\nIdeally what we should do is we should infer the search query from the entire\nconversation, and use that to perform the search.\n\nLet's implement an extra step in which the LLM is going to produce a query for\nus based on the entire chat history. That way we can be sure we're progressively\nworking on our query rather than rewriting it from scratch every time.\n\n\n.. edb:split-section::\n\n This is what we need to do: every time the user submits a message, we need to\n fetch the chat history, extract a search query from it using the LLM, and the\n other steps are going to the the same as before. Let's make the follwing\n modifications to the ``main.py``: first we need to create a function that\n prepares LLM inputs for the search query inference.\n\n\n .. code-block:: python\n :caption: app/main.py\n\n async def generate_search_query(\n query: str, message_history: list[GetMessagesResult]\n ) -> str:\n system_prompt = (\n \"You are a helpful assistant.\"\n + \" Your job is to extract a keyword search query\"\n + \" from a chat between an AI and a human.\"\n + \" Make sure it's a single most relevant keyword to maximize matching.\"\n + \" Only provide the query itself as your response.\"\n )\n\n formatted_history = \"\\n---\\n\".join(\n [\n f\"{message.role}: {message.body} (sources: {message.sources})\"\n for message in message_history\n ]\n )\n prompt = f\"Chat history: {formatted_history}\\n\\nUser message: {query} \\n\\n\"\n\n llm_response = get_llm_completion(\n system_prompt=system_prompt, messages=[{\"role\": \"user\", \"content\": prompt}]\n )\n\n return llm_response\n\n\n.. edb:split-section::\n\n And now we can use this function in ``post_messages`` in order to get our\n search query:\n\n\n .. code-block:: python-diff\n :caption: app/main.py\n\n class SearchResult(BaseModel):\n response: str | None = None\n + search_query: str | None = None\n sources: list[WebSource] | None = None\n\n\n @app.post(\"/messages\", status_code=HTTPStatus.CREATED)\n async def post_messages(\n search_terms: SearchTerms,\n username: str = Query(),\n chat_id: str = Query(),\n ) -> SearchResult:\n # 1. Fetch chat history\n chat_history = await get_messages_query(\n gel_client, username=username, chat_id=chat_id\n )\n\n # 2. Add incoming message to Gel\n _ = await add_message_query(\n gel_client,\n username=username,\n message_role=\"user\",\n message_body=search_terms.query,\n sources=[],\n chat_id=chat_id,\n )\n\n # 3. Generate a query and perform googling\n - search_query = search_terms.query\n + search_query = await generate_search_query(search_terms.query, chat_history)\n + web_sources = await search_web(search_query)\n\n\n # 5. Generate answer\n search_result = await generate_answer(\n search_terms.query,\n chat_history,\n web_sources,\n )\n + search_result.search_query = search_query # add search query to the output\n + # to see what the bot is searching for\n # 6. Add LLM response to Gel\n _ = await add_message_query(\n gel_client,\n username=username,\n message_role=\"assistant\",\n message_body=search_result.response,\n sources=[s.url for s in search_result.sources],\n chat_id=chat_id,\n )\n\n # 7. Send result back to the client\n return search_result\n\n\n.. edb:split-section::\n\n Done! We've now fully integrated the chat history into out app and enabled\n natural language conversations. As before, let's quickly test out the\n improvements before moving on:\n\n\n .. code-block:: bash\n\n $ curl -X 'POST' \\\n 'http://localhost:8000/messages?username=alice&chat_id=d4eed420-e903-11ef-b8a7-8718abdafbe1' \\\n -H 'accept: application/json' \\\n -H 'Content-Type: application/json' \\\n -d '{\n \"query\": \"what are people saying about gel\"\n }'\n\n $ curl -X 'POST' \\\n 'http://localhost:8000/messages?username=alice&chat_id=d4eed420-e903-11ef-b8a7-8718abdafbe1' \\\n -H 'accept: application/json' \\\n -H 'Content-Type: application/json' \\\n -d '{\n \"query\": \"do they like it or not\"\n }'\n\n\n6. Use Gel's advanced features to create a RAG\n==============================================\n\nAt this point we have a decent search bot that can refine a search query over\nmultiple turns of a conversation.\n\nIt's time to add the final touch: we can make the bot remember previous similar\ninteractions with the user using retrieval-augmented generation (RAG).\n\nTo achieve this we need to implement similarity search across message history:\nwe're going to create a vector embedding for every message in the database using\na neural network. Every time we generate a Google search query, we're also going\nto use it to search for similar messages in user's message history, and inject\nthe corresponding chat into the prompt. That way the search bot will be able to\nquickly \"remember\" similar interactions with the user and use them to understand\nwhat they are looking for.\n\nGel enables us to implement such a system with only minor modifications to the\nschema.\n\n\n.. edb:split-section::\n\n We begin by enabling the ``ai`` extension by adding the following like on top of\n the :dotgel:`dbschema/default`:\n\n .. code-block:: sdl-diff\n :caption: dbschema/default.esdl\n\n + using extension ai;\n\n\n.. edb:split-section::\n\n ... and do the migration:\n\n\n .. code-block:: bash\n\n $ gel migration create\n $ gel migrate\n\n\n.. edb:split-section::\n\n Next, we need to configure the API key in Gel for whatever embedding provider\n we're going to be using. As per documentation, let's open up the CLI by typing\n ``gel`` and run the following command (assuming we're using OpenAI):\n\n .. code-block:: edgeql-repl\n\n searchbot:main> configure current database\n insert ext::ai::OpenAIProviderConfig {\n secret := 'sk-....',\n };\n\n OK: CONFIGURE DATABASE\n\n\n.. edb:split-section::\n\n In order to get Gel to automatically keep track of creating and updating\n message embeddings, all we need to do is create a deferred index like this.\n Don't forget to run a migration one more time!\n\n .. code-block:: sdl-diff\n\n type Message {\n role: str;\n body: str;\n timestamp: datetime {\n default := datetime_current();\n }\n multi sources: str;\n\n + deferred index ext::ai::index(embedding_model := 'text-embedding-3-small')\n + on (.body);\n }\n\n\n.. edb:split-section::\n\n And we're done! Gel is going to cook in the background for a while and generate\n embedding vectors for our queries. To make sure nothing broke we can follow\n Gel's AI documentation and take a look at instance logs:\n\n .. code-block:: bash\n\n $ gel instance logs -I searchbot | grep api.openai.com\n\n INFO 50121 searchbot 2025-01-30T14:39:53.364 httpx: HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n\n\n.. edb:split-section::\n\n It's time to create the second half of the similarity search - the search query.\n The query needs to fetch ``k`` chats in which there're messages that are most\n similar to our current message. This can be a little difficult to visualize in\n your head, so here's the query itself:\n\n .. code-block:: edgeql\n :caption: app/queries/search_chats.edgeql\n\n with\n user := (select User filter .name = $username),\n chats := (\n select Chat\n filter .$current_chat_id\n )\n\n select chats {\n distance := min(\n ext::ai::search(\n .messages,\n >$embedding,\n ).distance,\n ),\n messages: {\n role, body, sources\n }\n }\n\n order by .distance\n limit $limit;\n\n\n.. edb:split-section::\n\n .. note::\n\n Before we can integrate this query into our Python app, we also need to add a\n new dependency for the Python binding: ``httpx-sse``. It's enables streaming\n outputs, which we're not going to use right now, but we won't be able to\n create the AI client without it.\n\n Let's place in in ``app/queries/search_chats.edgeql``, run the codegen and modify\n our ``post_messages`` endpoint to keep track of those similar chats.\n\n .. code-block:: python-diff\n :caption: app.main.py\n\n + from edgedb.ai import create_async_ai, AsyncEdgeDBAI\n + from .queries.search_chats_async_edgeql import (\n + search_chats as search_chats_query,\n + )\n\n class SearchResult(BaseModel):\n response: str | None = None\n search_query: str | None = None\n sources: list[WebSource] | None = None\n + similar_chats: list[str] | None = None\n\n\n @app.post(\"/messages\", status_code=HTTPStatus.CREATED)\n async def post_messages(\n search_terms: SearchTerms,\n username: str = Query(),\n chat_id: str = Query(),\n ) -> SearchResult:\n # 1. Fetch chat history\n chat_history = await get_messages_query(\n gel_client, username=username, chat_id=chat_id\n )\n\n # 2. Add incoming message to Gel\n _ = await add_message_query(\n gel_client,\n username=username,\n message_role=\"user\",\n message_body=search_terms.query,\n sources=[],\n chat_id=chat_id,\n )\n\n # 3. Generate a query and perform googling\n search_query = await generate_search_query(search_terms.query, chat_history)\n web_sources = await search_web(search_query)\n\n + # 4. Fetch similar chats\n + db_ai: AsyncEdgeDBAI = await create_async_ai(gel_client, model=\"gpt-4o-mini\")\n + embedding = await db_ai.generate_embeddings(\n + search_query, model=\"text-embedding-3-small\"\n + )\n + similar_chats = await search_chats_query(\n + gel_client,\n + username=username,\n + current_chat_id=chat_id,\n + embedding=embedding,\n + limit=1,\n + )\n\n # 5. Generate answer\n search_result = await generate_answer(\n search_terms.query,\n chat_history,\n web_sources,\n + similar_chats,\n )\n search_result.search_query = search_query # add search query to the output\n # to see what the bot is searching for\n # 6. Add LLM response to Gel\n _ = await add_message_query(\n gel_client,\n username=username,\n message_role=\"assistant\",\n message_body=search_result.response,\n sources=[s.url for s in search_result.sources],\n chat_id=chat_id,\n )\n\n # 7. Send result back to the client\n return search_result\n\n\n.. edb:split-section::\n\n Finally, the answer generator needs to get updated one more time, since we need\n to inject the additional messages into the prompt.\n\n .. code-block:: python-diff\n :caption: app/main.py\n\n async def generate_answer(\n query: str,\n chat_history: list[GetMessagesResult],\n web_sources: list[WebSource],\n + similar_chats: list[list[GetMessagesResult]],\n ) -> SearchResult:\n system_prompt = (\n \"You are a helpful assistant that answers user's questions\"\n + \" by finding relevant information in HackerNews threads.\"\n + \" When answering the question, describe conversations that people have around the subject, provided to you as a context, or say i don't know if they are completely irrelevant.\"\n + + \" You can reference previous conversation with the user that\"\n + + \" are provided to you, if they are relevant, by explicitly referring\"\n + + \" to them by saying as we discussed in the past.\"\n )\n\n prompt = f\"User search query: {query}\\n\\nWeb search results:\\n\"\n\n for i, source in enumerate(web_sources):\n prompt += f\"Result {i} (URL: {source.url}):\\n\"\n prompt += f\"{source.text}\\n\\n\"\n\n + prompt += \"Similar chats with the same user:\\n\"\n\n + formatted_chats = []\n + for i, chat in enumerate(similar_chats):\n + formatted_chat = f\"Chat {i}: \\n\"\n + for message in chat.messages:\n + formatted_chat += f\"{message.role}: {message.body}\\n\"\n + formatted_chats.append(formatted_chat)\n\n + prompt += \"\\n\".join(formatted_chats)\n\n messages = [\n {\"role\": message.role, \"content\": message.body} for message in chat_history\n ]\n messages.append({\"role\": \"user\", \"content\": prompt})\n\n llm_response = get_llm_completion(\n system_prompt=system_prompt,\n messages=messages,\n )\n\n search_result = SearchResult(\n response=llm_response,\n sources=web_sources,\n + similar_chats=formatted_chats,\n )\n\n return search_result\n\n\n.. edb:split-section::\n\n And one last time, let's check to make sure everything works:\n\n .. code-block:: bash\n\n $ curl -X 'POST' \\\n 'http://localhost:8000/messages?username=alice&chat_id=d4eed420-e903-11ef-b8a7-8718abdafbe1' \\\n -H 'accept: application/json' \\\n -H 'Content-Type: application/json' \\\n -d '{\n \"query\": \"remember that cool db i was talking to you about?\"\n }'\n\n\nKeep going!\n===========\n\nThis tutorial is over, but this app surely could use way more features!\n\nBasic functionality like deleting messages, a user interface or real web\nsearch, sure. But also authentication or access policies -- Gel will let you\nset those up in minutes.\n\nThanks!\n\n\n\n\n\n\n\n" }, { "path": "docs/intro/tutorials/gel_drizzle_booknotes.rst", "content": ".. _ref_guide_gel_drizzle_booknotes:\n\n====================================================\nBuild a Book Notes App with Drizzle\n====================================================\n\n:edb-alt-title: Building a book notes app using Gel, Drizzle ORM, and Next.js\n\nIn this tutorial we're going to walk you through building a Book Notes application\nthat lets you keep track of books you've read along with your personal notes.\nWe'll be using Gel as the database, Drizzle as the ORM layer, and Next.js as our\nfull-stack framework.\n\nGel is a data layer designed to supercharge PostgreSQL with a graph-like object model, access control, Auth, and many other features. It provides a unified schema and tooling experience across multiple languages, making it ideal for projects with diverse tech stacks. With Gel, you get access to EdgeQL, which eliminates n+1 query problems, supports automatic embeddings, and offers a seamless developer experience.\n\nDrizzle, on the other hand, is a TypeScript ORM that offers type safety and a great developer experience. By combining Gel with Drizzle, you can leverage Gel's powerful features while using Drizzle as a familiar ORM layer to interact with your database. This approach is perfect for developers who want to start learning Gel or prefer using Drizzle for their projects.\nexperience. Next.js is a React framework for building production-ready web applications with features like server components, built-in routing, and API routes. By the end of this tutorial, you will see how these technologies work together to create a modern, full-stack web application with a great developer experience.\n\n.. note::\n\n The complete source code for this tutorial is available in our `Gel Examples repository\n `_.\n\n\nWe will start by creating a Gel schema, setting up Drizzle, and then building\na Next.js application with API routes and a simple UI to manage your book collection and notes.\n\n\n1. Initialize the project\n=========================\n\n.. edb:split-section::\n\n Let's start with setting up our project. We'll create a new Next.js application,\n install the necessary dependencies, and initialize a Gel project.\n\n Here's a summary of the setup steps we'll follow:\n\n 1. Create a Next.js application\n 2. Install Gel and related packages\n 3. Initialize a Gel project\n 4. Update schema and apply migrations\n 5. Install and set up Drizzle\n 6. Pull schema into Drizzle\n 7. Configure hooks in gel.toml\n\n.. edb:split-section::\n\n First, let's create a new Next.js application. You can use the\n ``create-next-app`` command to set up a new project. When prompted,\n choose TypeScript, ESLint, Tailwind CSS, and the App Router. You can\n skip the default import alias configuration.\n This will create a new Next.js application with the necessary\n configuration files and dependencies.\n\n .. note::\n\n Make sure you have Node.js and npm installed on your machine.\n\n .. code-block:: bash\n\n # Step 1: Create a Next.js application\n $ npx create-next-app@latest book-notes-app\n\n # When prompted, choose:\n # āœ” Would you like to use TypeScript? Yes\n # āœ” Would you like to use ESLint? Yes\n # āœ” Would you like to use Tailwind CSS? Yes\n # āœ” Would you like to use `src/` directory? Yes\n # āœ” Would you like to use App Router? Yes\n # āœ” Would you like to use Turbopack for `next dev`? No\n # āœ” Would you like to customize the default import alias (@/*)? No\n\n $ cd book-notes-app\n\n.. edb:split-section::\n\n Next, let's install the Gel library. We'll need ``gel`` for database access.\n\n .. code-block:: bash\n\n $ npm i gel\n\n.. edb:split-section::\n\n Now, we'll initialize a Gel project. This will create the necessary configuration\n files and set up a local Gel instance.\n\n .. code-block:: bash\n\n $ npx gel project init\n\n2. Define the Gel schema\n========================\n\n\nNow that we have our project environment set up, let's define our database schema.\nFor our Book Notes app, we'll create two main types:\n\n1. ``Book`` - to store information about books\n2. ``Note`` - to store notes associated with each book\n\nLet's edit the :dotgel:`dbschema/default` file that was created during initialization.\n\n.. edb:split-section::\n\n Our schema defines two types:\n\n - ``Book`` with properties like title, author, publication year, genre, and read date.\n - ``Note`` with text content and a timestamp, linked to a specific book.\n\n The relationship is defined such that a book can have multiple notes, and\n each note belongs to exactly one book. We're using a computed link ``notes`` to\n allow easy access to a book's notes.\n\n .. code-block:: sdl\n :caption: :dotgel:`dbschema/default`\n\n module default {\n type Book {\n required title: str;\n author: str;\n year: int16;\n genre: str;\n read_date: datetime;\n\n # Relationship to notes\n multi notes := . [run ``drizzle-kit pull``].\n\n\n .. code-block:: typescript\n :caption: drizzle/schema.ts\n :class: collapsible\n\n import { gelTable, uniqueIndex, uuid, text, timestamptz, smallint, foreignKey } from \"drizzle-orm/gel-core\"\n import { sql } from \"drizzle-orm\"\n\n\n export const book = gelTable(\"Book\", {\n \tid: uuid().default(sql`uuid_generate_v4()`).primaryKey().notNull(),\n \tauthor: text(),\n \tgenre: text(),\n \treadDate: timestamptz(\"read_date\"),\n \ttitle: text().notNull(),\n \tyear: smallint(),\n }, (table) => [\n \tuniqueIndex(\"5f1d3546-1943-11f0-be08-df1707d45eaa;schemaconstr\").using(\"btree\", table.id.asc().nullsLast().op(\"uuid_ops\")),\n ]);\n\n export const note = gelTable(\"Note\", {\n \tid: uuid().default(sql`uuid_generate_v4()`).primaryKey().notNull(),\n \tbookId: uuid(\"book_id\").notNull(),\n \tcreatedAt: timestamptz(\"created_at\").default(sql`(clock_timestamp())`),\n \ttext: text().notNull(),\n }, (table) => [\n \tuniqueIndex(\"5f1e4652-1943-11f0-a4a0-f1f912666606;schemaconstr\").using(\"btree\", table.id.asc().nullsLast().op(\"uuid_ops\")),\n \tforeignKey({\n \t\tcolumns: [table.bookId],\n \t\tforeignColumns: [book.id],\n \t\tname: \"Note_fk_book\"\n \t}),\n ]);\n\n\n.. edb:split-section::\n\n Finally, we need to update the hooks in our ``gel.toml`` file to ensure that our\n Drizzle schema stays in sync with our Gel schema. Every time we apply a migration,\n we want to run the Drizzle pull command to update the TypeScript files.\n\n .. code-block:: toml-diff\n :caption: gel.toml\n\n + [hooks]\n + after_migration_apply = [\n + \"npx drizzle-kit pull\"\n + ]\n\n\n4. Creating the database client\n================================\n\n.. edb:split-section::\n\n Now that we have our schema set up, let's create a database client that we can use\n throughout our application. This client will connect to our Gel database using\n Drizzle.\n\n .. code-block:: typescript\n :caption: src/db/index.ts\n\n import { drizzle } from 'drizzle-orm/gel';\n import { createClient } from 'gel';\n\n import * as schema from '../../drizzle/schema';\n import * as relations from '../../drizzle/relations';\n\n // Initialize Gel client\n const gelClient = createClient();\n\n // Create Drizzle instance\n export const db = drizzle({ client: gelClient, schema: {\n ...schema,\n ...relations,\n } });\n\n // Helper types for use in our application\n export type Book = typeof schema.book.$inferSelect;\n export type NewBook = typeof schema.book.$inferInsert;\n export interface BookWithNotes extends Book {\n notes: Note[];\n };\n\n export type Note = typeof schema.note.$inferSelect;\n export type NewNote = typeof schema.note.$inferInsert;\n\n5. Implementing API Routes\n===========================\n\nNext, let's implement the API routes for our book notes application. With Next.js, we can create API endpoints in the ``app/api`` directory to handle HTTP requests.\n\n.. edb:split-section::\n\n We'll start by creating a route for managing all books. This will handle\n fetching all books and adding new books. The ``GET`` method will return a list\n of all books, while the ``POST`` method will allow us to add a new book.\n We'll also include error handling for both methods. In both, we'll use\n Drizzle ORM to interact with the database.\n\n .. code-block:: typescript\n :caption: app/api/books/route.ts\n\n import { NextResponse } from 'next/server';\n import { db } from '@/src/db';\n import { books } from '@/drizzle/schema';\n\n export async function GET() {\n try {\n const allBooks = await db.query.book.findMany({\n with: {\n notes: true,\n },\n });\n\n return NextResponse.json(allBooks);\n } catch (error) {\n console.error('Error fetching books:', error);\n return NextResponse.json(\n { error: 'Failed to fetch books' },\n { status: 500 }\n );\n }\n }\n\n export async function POST(request: Request) {\n try {\n const body = await request.json();\n\n const result = await db.insert(book).values({\n title: body.title,\n author: body.author,\n year: body.year,\n genre: body.genre,\n readDate: new Date(body.read_date),\n }).returning();\n\n return NextResponse.json(result[0], { status: 201 });\n } catch (error) {\n console.error('Error adding book:', error);\n return NextResponse.json(\n { error: 'Failed to add book' },\n { status: 500 }\n );\n }\n }\n\n.. edb:split-section::\n\n Next, let's create a route for managing a specific book by its ID. This will handle\n getting book details, updating books, and deleting books.\n\n - ``GET`` method will fetch a specific book by its ID.\n - ``PUT`` method will update the book details based on the request body.\n - ``DELETE`` method will delete the book and all its associated notes.\n\n We'll also include error handling for each method.\n\n .. code-block:: typescript\n :caption: src/app/api/books/[id]/route.ts\n\n import { NextResponse } from 'next/server';\n import { db } from '@/src/db';\n import { book, note } from '@/drizzle/schema';\n import { eq } from 'drizzle-orm';\n\n export async function GET(\n request: Request,\n { params }: { params: Promise<{ id: string }> }\n ) {\n const { id } = await params;\n try {\n const requestedBook = await db.query.book.findFirst({\n where: eq(books.id, id),\n with: {\n note: true,\n },\n });\n\n if (!requestedBook) {\n return NextResponse.json(\n { error: 'Book not found' },\n { status: 404 }\n );\n }\n\n return NextResponse.json(requestedBook);\n } catch (error) {\n console.error('Error fetching book:', error);\n return NextResponse.json(\n { error: 'Failed to fetch book' },\n { status: 500 }\n );\n }\n }\n\n export async function PUT(\n request: Request,\n { params }: { params: Promise<{ id: string }> }\n ) {\n const { id } = await params;\n try {\n const body = await request.json();\n\n const result = await db.update(book)\n .set({\n title: body.title,\n author: body.author,\n year: body.year,\n genre: body.genre,\n readDate: new Date(body.read_date),\n })\n .where(eq(books.id, id))\n .returning();\n\n if (result.length === 0) {\n return NextResponse.json(\n { error: 'Book not found' },\n { status: 404 }\n );\n }\n\n return NextResponse.json(result[0]);\n } catch (error) {\n console.error('Error updating book:', error);\n return NextResponse.json(\n { error: 'Failed to update book' },\n { status: 500 }\n );\n }\n }\n\n export async function DELETE(\n request: Request,\n { params }: { params: Promise<{ id: string }> }\n ) {\n const { id } = await params;\n try {\n // First delete associated notes\n await db.delete(note).where(eq(note.bookId, id));\n\n // Then delete the book\n const result = await db.delete(book)\n .where(eq(book.id, id))\n .returning();\n\n if (result.length === 0) {\n return NextResponse.json(\n { error: 'Book not found' },\n { status: 404 }\n );\n }\n\n return NextResponse.json({ success: true });\n } catch (error) {\n console.error('Error deleting book:', error);\n return NextResponse.json(\n { error: 'Failed to delete book' },\n { status: 500 }\n );\n }\n }\n\n.. edb:split-section::\n\n Now, let's create a route for adding notes to a book. This endpoint will handle the\n creation of new notes for a specific book. The ``POST`` method will accept\n a request body with the note text and the book ID.\n\n .. code-block:: typescript\n :caption: src/app/api/books/[id]/notes/route.ts\n\n import { NextResponse } from 'next/server';\n import { db } from '@/src/db';\n import { note } from '@/drizzle/schema';\n\n export async function POST(\n request: Request,\n { params }: { params: Promise<{ id: string }> }\n ) {\n const { id } = await params;\n try {\n const body = await request.json();\n\n const result = await db.insert(note).values({\n text: body.text,\n bookId: id,\n }).returning();\n\n return NextResponse.json(result[0], { status: 201 });\n } catch (error) {\n console.error('Error adding note:', error);\n return NextResponse.json(\n { error: 'Failed to add note' },\n { status: 500 }\n );\n }\n }\n\n.. edb:split-section::\n\n Finally, let's create a route for updating and deleting individual notes.\n This will handle the ``PUT`` and ``DELETE`` methods for a specific note.\n The ``PUT`` method will update the note text, while the ``DELETE`` method\n will delete the note.\n\n .. code-block:: typescript\n :caption: src/app/api/notes/[id]/route.ts\n\n import { NextResponse } from 'next/server';\n import { db } from '@/src/db';\n import { note } from '@/drizzle/schema';\n import { eq } from 'drizzle-orm';\n\n export async function PUT(\n request: Request,\n { params }: { params: Promise<{ id: string }> }\n ) {\n const { id } = await params;\n try {\n const body = await request.json();\n\n const result = await db.update(note)\n .set({\n text: body.text,\n })\n .where(eq(note.id, id))\n .returning();\n\n if (result.length === 0) {\n return NextResponse.json(\n { error: 'Note not found' },\n { status: 404 }\n );\n }\n\n return NextResponse.json(result[0]);\n } catch (error) {\n console.error('Error updating note:', error);\n return NextResponse.json(\n { error: 'Failed to update note' },\n { status: 500 }\n );\n }\n }\n\n export async function DELETE(\n request: Request,\n { params }: { params: Promise<{ id: string }> }\n ) {\n const { id } = await params;\n try {\n const result = await db.delete(note)\n .where(eq(notes.id, id))\n .returning();\n\n if (result.length === 0) {\n return NextResponse.json(\n { error: 'Note not found' },\n { status: 404 }\n );\n }\n\n return NextResponse.json({ success: true });\n } catch (error) {\n console.error('Error deleting note:', error);\n return NextResponse.json(\n { error: 'Failed to delete note' },\n { status: 500 }\n );\n }\n }\n\n.. edb:split-section::\n\n We can test our API routes using a tool like Postman or cURL. Let's start the\n development server and test the routes.\n\n .. code-block:: bash\n\n $ npm run dev\n\n.. edb:split-section::\n\n You can now access the API routes at ``http://localhost:3000/api`` (or the port specified in your environment). For example, to access the books route, you can go to ``http://localhost:3000/api/books``. You can use Postman or cURL to test the endpoints. For example, to fetch all books, you can use the following cURL command:\n\n .. code-block:: bash\n\n $ curl -X GET http://localhost:3000/api/books\n\n.. edb:split-section::\n\n To add a new book, you can use the following cURL command:\n\n .. code-block:: bash\n\n $ curl -X POST http://localhost:3000/api/books \\\n -H \"Content-Type: application/json\" \\\n -d '{\"title\": \"The Great Gatsby\", \"author\": \"F. Scott Fitzgerald\", \"year\": 1925, \"genre\": \"Fiction\", \"read_date\": \"2023-10-01\"}'\n\n.. edb:split-section::\n\n Or to create a new note for a book, you can use the following cURL command\n (replace ```` with the actual book ID):\n\n .. code-block:: bash\n\n $ curl -X POST http://localhost:3000/api/books//notes \\\n -H \"Content-Type: application/json\" \\\n -d '{\"text\": \"This is a great book!\"}'\n\n6. Building the UI\n==================\n\nNow that we have our API routes in place, we can build a user interface for our book\nnotes application. We'll use Tailwind CSS for styling, which was included when we\ncreated our Next.js application.\n\nWe won't go into extensive UI details, but here's a basic implementation for the\nhome page that lists all books.\n\n.. edb:split-section::\n\n We'll start by creating a home page that fetches and displays all books from our API.\n This page will also include a link to add a new book.\n We'll use the ``useEffect`` hook to fetch the books when the component mounts.\n We'll also handle loading states and error handling.\n The home page will display a list of books with their titles, authors, publication years,\n genres, and the number of notes associated with each book.\n\n .. code-block:: typescript\n :caption: app/page.tsx\n :class: collapsible\n\n 'use client';\n\n import { useState, useEffect } from 'react';\n import Link from 'next/link';\n import { BookWithNotes } from '@/db';\n\n export default function Home() {\n const [books, setBooks] = useState([]);\n const [loading, setLoading] = useState(true);\n\n useEffect(() => {\n async function fetchBooks() {\n try {\n const response = await fetch('/api/books');\n if (!response.ok) throw new Error('Failed to fetch books');\n const data = await response.json();\n setBooks(data);\n } catch (error) {\n console.error('Error:', error);\n } finally {\n setLoading(false);\n }\n }\n\n fetchBooks();\n }, []);\n\n if (loading) {\n return (\n
\n

Loading...

\n
\n );\n }\n\n return (\n
\n

\n My Book Notes\n

\n\n \n Add New Book\n \n\n
\n {books.length === 0 ? (\n

No books found. Add your first book!

\n ) : (\n books.map((book) => (\n \n \n

{book.title}

\n {book.author && (\n

\n by {book.author}\n

\n )}\n {book.year && (\n

\n Published: {book.year}\n

\n )}\n {book.genre && (\n

\n Genre: {book.genre}\n

\n )}\n

\n {book.notes?.length || 0} notes\n

\n\n
\n \n ))\n )}\n
\n \n );\n }\n\n.. edb:split-section::\n\n Next, let's create a form component for adding and editing books. This will be used in\n both the \"Add Book\" page and the \"Edit Book\" page.\n\n .. code-block:: typescript\n :caption: src/components/BookForm.tsx\n :class: collapsible\n\n 'use client';\n\n import { useState, FormEvent } from 'react';\n import { useRouter } from 'next/navigation';\n import { Book } from '../db';\n\n interface BookFormProps {\n book?: Book;\n isEditing?: boolean;\n }\n\n export default function BookForm({ book, isEditing = false }: BookFormProps) {\n const router = useRouter();\n const [title, setTitle] = useState(book?.title || '');\n const [author, setAuthor] = useState(book?.author || '');\n const [year, setYear] = useState(book?.year?.toString() || '');\n const [genre, setGenre] = useState(book?.genre || '');\n const [readDate, setReadDate] = useState(\n book?.readDate\n ? new Date(book.readDate).toISOString().split('T')[0]\n : ''\n );\n\n const handleSubmit = async (e: FormEvent) => {\n e.preventDefault();\n\n const bookData = {\n title,\n author,\n year: year ? parseInt(year) : undefined,\n genre,\n read_date: readDate || undefined,\n };\n\n try {\n if (isEditing && book) {\n // Update existing book\n await fetch(`/api/books/${book.id}`, {\n method: 'PUT',\n headers: { 'Content-Type': 'application/json' },\n body: JSON.stringify(bookData),\n });\n } else {\n // Create new book\n await fetch('/api/books', {\n method: 'POST',\n headers: { 'Content-Type': 'application/json' },\n body: JSON.stringify(bookData),\n });\n }\n\n router.push('/');\n router.refresh();\n } catch (error) {\n console.error('Error saving book:', error);\n }\n };\n\n return (\n
\n
\n \n setTitle(e.target.value)}\n required\n className=\"w-full px-4 py-2 bg-gray-800 text-white border border-gray-700 rounded focus:outline-none focus:ring-2 focus:ring-blue-500 transition duration-150\"\n />\n
\n\n
\n \n setAuthor(e.target.value)}\n className=\"w-full px-4 py-2 bg-gray-800 text-white border border-gray-700 rounded focus:outline-none focus:ring-2 focus:ring-blue-500 transition duration-150\"\n />\n
\n\n
\n \n setYear(e.target.value)}\n className=\"w-full px-4 py-2 bg-gray-800 text-white border border-gray-700 rounded focus:outline-none focus:ring-2 focus:ring-blue-500 transition duration-150\"\n />\n
\n\n
\n \n setGenre(e.target.value)}\n className=\"w-full px-4 py-2 bg-gray-800 text-white border border-gray-700 rounded focus:outline-none focus:ring-2 focus:ring-blue-500 transition duration-150\"\n />\n
\n\n
\n \n setReadDate(e.target.value)}\n className=\"w-full px-4 py-2 bg-gray-800 text-white border border-gray-700 rounded focus:outline-none focus:ring-2 focus:ring-blue-500 transition duration-150\"\n />\n
\n\n
\n router.back()}\n className=\"px-4 py-2 border border-gray-600 text-white rounded hover:bg-gray-700 transition duration-150\"\n >\n Cancel\n \n \n {isEditing ? 'Update Book' : 'Add Book'}\n \n
\n
\n );\n }\n\n.. edb:split-section::\n\n Now, let's create the \"Add Book\" page that uses our form component.\n\n .. code-block:: typescript\n :caption: app/books/add/page.tsx\n\n 'use client';\n\n import BookForm from \"@/src/components/BookForm\";\n\n export default function AddBookPage() {\n return (\n
\n

Add New Book

\n \n
\n );\n }\n\n.. edb:split-section::\n\n Let's also create a page to view book details and manage notes.\n\n .. code-block:: typescript\n :caption: src/app/books/[id]/page.tsx\n :class: collapsible\n\n 'use client';\n\n import { useState, useEffect, FormEvent, use } from 'react';\n import { useRouter } from 'next/navigation';\n import Link from 'next/link';\n import { BookWithNotes } from '@/src/db';\n\n export default function BookDetailPage({ params }: { params: Promise<{ id: string }> }) {\n const { id } = use(params);\n const router = useRouter();\n const [book, setBook] = useState(null);\n const [loading, setLoading] = useState(true);\n const [noteText, setNoteText] = useState('');\n\n useEffect(() => {\n async function fetchBook() {\n try {\n const response = await fetch(`/api/books/${id}`);\n if (!response.ok) {\n if (response.status === 404) {\n router.push('/');\n return;\n }\n throw new Error('Failed to fetch book');\n }\n const data = await response.json();\n setBook(data);\n } catch (error) {\n console.error('Error:', error);\n } finally {\n setLoading(false);\n }\n }\n\n fetchBook();\n }, [id, router]);\n\n const handleAddNote = async (e: FormEvent) => {\n e.preventDefault();\n\n if (!noteText.trim()) return;\n\n try {\n const response = await fetch(`/api/books/${id}/notes`, {\n method: 'POST',\n headers: { 'Content-Type': 'application/json' },\n body: JSON.stringify({ text: noteText }),\n });\n\n if (!response.ok) throw new Error('Failed to add note');\n\n const newNote = await response.json();\n setBook(prev => prev ? {\n ...prev,\n notes: [...prev.notes, newNote]\n } : null);\n setNoteText('');\n } catch (error) {\n console.error('Error adding note:', error);\n }\n };\n\n const handleDeleteNote = async (noteId: string) => {\n try {\n const response = await fetch(`/api/notes/${noteId}`, {\n method: 'DELETE',\n });\n\n if (!response.ok) throw new Error('Failed to delete note');\n\n setBook(prev => prev ? {\n ...prev,\n notes: prev.notes.filter(note => note.id !== noteId)\n } : null);\n } catch (error) {\n console.error('Error deleting note:', error);\n }\n };\n\n const handleDeleteBook = async () => {\n if (!confirm('Are you sure you want to delete this book and all its notes?')) {\n return;\n }\n\n try {\n const response = await fetch(`/api/books/${id}`, {\n method: 'DELETE',\n });\n\n if (!response.ok) throw new Error('Failed to delete book');\n\n router.push('/');\n } catch (error) {\n console.error('Error deleting book:', error);\n }\n };\n\n if (loading) {\n return (\n
\n

Loading...

\n
\n );\n }\n\n if (!book) {\n return (\n
\n

Book not found.

\n \n Back to All Books\n \n
\n );\n }\n\n return (\n
\n
\n \n ā† Back\n \n
\n\n
\n

\n {book.title}\n

\n
\n \n Edit\n \n \n Delete\n \n
\n
\n\n
\n {book.author && (\n

\n by {book.author}\n

\n )}\n
\n {book.year &&

Published: {book.year}

}\n {book.genre &&

Genre: {book.genre}

}\n {book.readDate && (\n

Read on: {new Date(book.readDate).toLocaleDateString()}

\n )}\n
\n
\n\n
\n

Notes

\n
\n
\n setNoteText(e.target.value)}\n placeholder=\"Add a new note...\"\n className=\"flex-grow px-4 py-2 bg-gray-700 text-white placeholder-gray-400 border border-gray-600 rounded-l focus:outline-none focus:ring-2 focus:ring-blue-500 transition duration-150\"\n />\n \n Add\n \n
\n
\n\n {book.notes.length === 0 ? (\n

\n No notes yet. Add your first note above.\n

\n ) : (\n
    \n {book.notes.map((note) => (\n \n
    \n

    {note.text}

    \n {note.createdAt && (\n

    \n {new Date(note.createdAt).toLocaleString()}\n

    \n )}\n
    \n handleDeleteNote(note.id)}\n className=\"text-red-400 hover:text-red-600 focus:outline-none transition duration-150\"\n >\n Delete\n \n \n ))}\n
\n )}\n
\n
\n );\n }\n\n.. edb:split-section::\n\n For a complete application, you would also need to implement an edit page for books.\n Here's a simplified example:\n\n .. code-block:: typescript\n :caption: src/app/books/[id]/edit/page.tsx\n :class: collapsible\n\n 'use client';\n\n import { useState, useEffect, use } from 'react';\n import { useRouter } from 'next/navigation';\n import { Book } from '@/src/db';\n import BookForm from '@/src/components/BookForm';\n\n export default function EditBookPage({ params }: { params: Promise<{ id: string }> }) {\n const router = useRouter();\n const { id } = use(params);\n const [book, setBook] = useState(null);\n const [loading, setLoading] = useState(true);\n\n useEffect(() => {\n async function fetchBook() {\n try {\n const response = await fetch(`/api/books/${id}`);\n if (!response.ok) {\n if (response.status === 404) {\n router.push('/');\n return;\n }\n throw new Error('Failed to fetch book');\n }\n const data = await response.json();\n setBook(data);\n } catch (error) {\n console.error('Error:', error);\n } finally {\n setLoading(false);\n }\n }\n\n fetchBook();\n }, [id, router]);\n\n if (loading) {\n return (\n
\n

Loading...

\n
\n );\n }\n\n if (!book) {\n return (\n
\n

Book not found.

\n \n
\n );\n }\n\n return (\n
\n

Edit Book

\n \n
\n );\n }\n\n.. edb:split-section::\n\n These UI components provide a basic but functional user interface for our Book Notes\n application. Tailwind CSS helps us create a clean and responsive design with minimal\n effort.\n\n Since we're focusing on the Gel and Drizzle integration, we won't detail every UI\n component, but the pattern is consistent throughout the application:\n\n - We use React hooks for state management (useState, useEffect)\n - We call our API endpoints to fetch and modify data\n - We use Tailwind CSS classes for styling the components\n - We implement client-side navigation with Next.js's useRouter\n\n7. Testing the application\n===========================\n\n.. edb:split-section::\n\n Now that we have built our API routes and basic UI, let's test our application.\n Start the development server:\n\n .. code-block:: bash\n\n $ npm run dev\n\n.. edb:split-section::\n\n Navigate to http://localhost:3000 in your browser, and you should see your Book Notes\n application. Try performing these operations to ensure everything is working correctly:\n\n 1. Adding a new book\n 2. Viewing book details\n 3. Adding notes to a book\n 4. Editing book information\n 5. Deleting notes\n 6. Deleting a book (which should also delete its notes)\n\n If you encounter any issues, check your browser's developer console and the terminal\n running your Next.js server for error messages.\n\n8. Next steps\n==============\n\nCongratulations! You've built a Book Notes application using Gel, Drizzle, and Next.js.\nThis tutorial demonstrated how these technologies can work together to create a\nfull-stack application.\n\nHere are some ideas for extending the application:\n\n1. **Add authentication**: Implement user authentication to allow multiple users\n to have their own book collections.\n\n2. **Advanced filtering**: Add the ability to filter books by genre, author, or\n reading status.\n\n3. **Book statistics**: Create a dashboard with statistics about your reading\n habits.\n\n4. **Reading goals**: Implement a feature to set and track reading goals.\n\n5. **Book recommendations**: Add a feature to recommend books based on what\n you've already read.\n\n6. **Import/Export**: Allow users to import or export their book data.\n\n7. **Search functionality**: Implement full-text search across books and notes.\n\nTo further explore the capabilities of Gel and Drizzle, you can check out these resources:\n\n- `Gel Documentation `_\n- `Drizzle ORM Documentation `_\n- `Next.js Documentation `_\n\nRemember, you can find the complete source code for this tutorial in our\n`Gel Examples repository `_.\n\nHappy coding!\n" }, { "path": "docs/intro/tutorials/index.rst", "content": "=========\nTutorials\n=========\n\n.. toctree::\n :maxdepth: 2\n\n ai_fastapi_searchbot\n gel_drizzle_booknotes\n" }, { "path": "docs/redirects", "content": "/guides/cloud -> /cloud\n\n/cli/edgedb -> docs/cli/gel\n/cli/edgedb_analyze -> docs/cli/gel_analyze\n/cli/edgedb_branch/edgedb_branch_create -> docs/cli/gel_branch/gel_branch_create\n/cli/edgedb_branch/edgedb_branch_drop -> docs/cli/gel_branch/gel_branch_drop\n/cli/edgedb_branch/edgedb_branch_list -> docs/cli/gel_branch/gel_branch_list\n/cli/edgedb_branch/edgedb_branch_merge -> docs/cli/gel_branch/gel_branch_merge\n/cli/edgedb_branch/edgedb_branch_rebase -> docs/cli/gel_branch/gel_branch_rebase\n/cli/edgedb_branch/edgedb_branch_rename -> docs/cli/gel_branch/gel_branch_rename\n/cli/edgedb_branch/edgedb_branch_switch -> docs/cli/gel_branch/gel_branch_switch\n/cli/edgedb_branch/edgedb_branch_wipe -> docs/cli/gel_branch/gel_branch_wipe\n/cli/edgedb_branch/index -> docs/cli/gel_branch/index\n/cli/edgedb_cli_upgrade -> docs/cli/gel_cli_upgrade\n/cli/edgedb_cloud/edgedb_cloud_login -> docs/cli/gel_cloud/gel_cloud_login\n/cli/edgedb_cloud/edgedb_cloud_logout -> docs/cli/gel_cloud/gel_cloud_logout\n/cli/edgedb_cloud/edgedb_cloud_secretkey/edgedb_cloud_secretkey_create -> docs/cli/gel_cloud/gel_cloud_secretkey/edgedb_cloud_secretkey_create\n/cli/edgedb_cloud/edgedb_cloud_secretkey/edgedb_cloud_secretkey_list -> docs/cli/gel_cloud/gel_cloud_secretkey/edgedb_cloud_secretkey_list\n/cli/edgedb_cloud/edgedb_cloud_secretkey/edgedb_cloud_secretkey_revoke -> docs/cli/gel_cloud/gel_cloud_secretkey/edgedb_cloud_secretkey_revoke\n/cli/edgedb_cloud/edgedb_cloud_secretkey/index -> docs/cli/gel_cloud/gel_cloud_secretkey/index\n/cli/edgedb_cloud/index -> docs/cli/gel_cloud/index\n/cli/edgedb_configure -> docs/cli/gel_configure\n/cli/edgedb_connopts -> docs/cli/gel_connopts\n/cli/edgedb_database/edgedb_database_create -> docs/cli/gel_database/gel_database_create\n/cli/edgedb_database/edgedb_database_drop -> docs/cli/gel_database/gel_database_drop\n/cli/edgedb_database/edgedb_database_wipe -> docs/cli/gel_database/gel_database_wipe\n/cli/edgedb_database/index -> docs/cli/gel_database/index\n/cli/edgedb_describe/edgedb_describe_object -> docs/cli/gel_describe/gel_describe_object\n/cli/edgedb_describe/edgedb_describe_schema -> docs/cli/gel_describe/gel_describe_schema\n/cli/edgedb_describe/index -> docs/cli/gel_describe/index\n/cli/edgedb_dump -> docs/cli/gel_dump\n/cli/edgedb_info -> docs/cli/gel_info\n/cli/edgedb_instance/edgedb_instance_create -> docs/cli/gel_instance/gel_instance_create\n/cli/edgedb_instance/edgedb_instance_credentials -> docs/cli/gel_instance/gel_instance_credentials\n/cli/edgedb_instance/edgedb_instance_destroy -> docs/cli/gel_instance/gel_instance_destroy\n/cli/edgedb_instance/edgedb_instance_link -> docs/cli/gel_instance/gel_instance_link\n/cli/edgedb_instance/edgedb_instance_list -> docs/cli/gel_instance/gel_instance_list\n/cli/edgedb_instance/edgedb_instance_logs -> docs/cli/gel_instance/gel_instance_logs\n/cli/edgedb_instance/edgedb_instance_reset_password -> docs/cli/gel_instance/gel_instance_reset_password\n/cli/edgedb_instance/edgedb_instance_restart -> docs/cli/gel_instance/gel_instance_restart\n/cli/edgedb_instance/edgedb_instance_revert -> docs/cli/gel_instance/gel_instance_revert\n/cli/edgedb_instance/edgedb_instance_start -> docs/cli/gel_instance/gel_instance_start\n/cli/edgedb_instance/edgedb_instance_status -> docs/cli/gel_instance/gel_instance_status\n/cli/edgedb_instance/edgedb_instance_stop -> docs/cli/gel_instance/gel_instance_stop\n/cli/edgedb_instance/edgedb_instance_unlink -> docs/cli/gel_instance/gel_instance_unlink\n/cli/edgedb_instance/edgedb_instance_upgrade -> docs/cli/gel_instance/gel_instance_upgrade\n/cli/edgedb_instance/index -> docs/cli/gel_instance/index\n/cli/edgedb_list -> docs/cli/gel_list\n/cli/edgedb_migrate -> docs/cli/gel_migrate\n/cli/edgedb_migration/edgedb_migration_apply -> docs/cli/gel_migration/gel_migration_apply\n/cli/edgedb_migration/edgedb_migration_create -> docs/cli/gel_migration/gel_migration_create\n/cli/edgedb_migration/edgedb_migration_edit -> docs/cli/gel_migration/gel_migration_edit\n/cli/edgedb_migration/edgedb_migration_extract -> docs/cli/gel_migration/gel_migration_extract\n/cli/edgedb_migration/edgedb_migration_log -> docs/cli/gel_migration/gel_migration_log\n/cli/edgedb_migration/edgedb_migration_status -> docs/cli/gel_migration/gel_migration_status\n/cli/edgedb_migration/edgedb_migration_upgrade_check -> docs/cli/gel_migration/gel_migration_upgrade_check\n/cli/edgedb_migration/index -> docs/cli/gel_migration/index\n/cli/edgedb_project/edgedb_project_info -> docs/cli/gel_project/gel_project_info\n/cli/edgedb_project/edgedb_project_init -> docs/cli/gel_project/gel_project_init\n/cli/edgedb_project/edgedb_project_unlink -> docs/cli/gel_project/gel_project_unlink\n/cli/edgedb_project/edgedb_project_upgrade -> docs/cli/gel_project/gel_project_upgrade\n/cli/edgedb_project/index -> docs/cli/gel_project/index\n/cli/edgedb_query -> docs/cli/gel_query\n/cli/edgedb_restore -> docs/cli/gel_restore\n/cli/edgedb_server/edgedb_server_info -> docs/cli/gel_server/gel_server_info\n/cli/edgedb_server/edgedb_server_install -> docs/cli/gel_server/gel_server_install\n/cli/edgedb_server/edgedb_server_list_versions -> docs/cli/gel_server/gel_server_list_versions\n/cli/edgedb_server/edgedb_server_uninstall -> docs/cli/gel_server/gel_server_uninstall\n/cli/edgedb_server/index -> docs/cli/gel_server/index\n/cli/edgedb_ui -> docs/cli/gel_ui\n/cli/edgedb_watch -> docs/cli/gel_watch\n" }, { "path": "docs/redirects.js", "content": "// See https://nextjs.org/docs/app/api-reference/config/next-config-js/redirects\nmodule.exports = [\n {\n source: \"/changelog/:path*\",\n destination: \"/resources/changelog/:path*\",\n permanent: false,\n },\n {\n source: \"/guides/cheatsheet/:path*\",\n destination: \"/resources/cheatsheets/:path*\",\n permanent: true,\n },\n {\n source: \"/guides/ai/:path*\",\n destination: \"/ai/:path*\",\n permanent: true,\n },\n {\n source: \"/changelog/6_x\",\n destination: \"/resources/changelog/6_x\",\n permanent: true,\n },\n {\n source: \"/database/:path*\",\n destination: \"/reference/:path*\",\n permanent: false,\n },\n {\n source: \"/guides/:path*\",\n destination: \"/resources/guides/:path*\",\n permanent: false,\n },\n {\n source: \"/reference/reference/bindings/datetime\",\n destination: \"/reference/using/datetime\",\n permanent: false,\n },\n {\n source: \"/reference/reference/bindings\",\n destination: \"/reference/using\",\n permanent: false,\n },\n {\n source: \"/reference/clients/go/:path*\",\n destination: \"https://pkg.go.dev/github.com/geldata/gel-go\",\n permanent: false,\n },\n {\n source: \"/reference/clients/rust/:path*\",\n destination: \"https://docs.rs/gel-tokio\",\n permanent: false,\n },\n {\n source: \"/reference/libraries/dotnet/:path*\",\n destination: \"https://github.com/geldata/gel-net\",\n permanent: false,\n },\n {\n source: \"/reference/libraries/elixir/:path*\",\n destination: \"https://hexdocs.pm/gel\",\n permanent: false,\n },\n {\n source: \"/reference/libraries/java/:path*\",\n destination: \"https://github.com/geldata/gel-java\",\n permanent: false,\n },\n // Use the further redirects to get to the correct point in the consolidated docs\n {\n source: \"/reference/libraries/js/:path*\",\n destination: \"/reference/clients/js/:path*\",\n permanent: false,\n },\n {\n source: \"/reference/libraries/python/:path*\",\n destination: \"/reference/clients/python/:path*\",\n permanent: false,\n },\n {\n source: \"/reference/libraries/:path*\",\n destination: \"/reference/using\",\n permanent: false,\n },\n {\n source: \"/reference/clients/js/delete#delete\",\n destination: \"/reference/using/js/querybuilder\",\n permanent: false,\n },\n {\n source: \"/reference/clients/js/driver\",\n destination: \"/reference/using/js\",\n permanent: false,\n },\n {\n source: \"/reference/clients/js/for\",\n destination: \"/reference/using/js/querybuilder#for\",\n permanent: false,\n },\n {\n source: \"/reference/clients/js/funcops\",\n destination: \"/reference/using/js/querybuilder#functions-and-operators\",\n permanent: false,\n },\n {\n source: \"/reference/clients/js/group\",\n destination: \"/reference/using/js/querybuilder#group\",\n permanent: false,\n },\n {\n source: \"/reference/clients/js/insert\",\n destination: \"/reference/using/js/querybuilder#insert\",\n permanent: false,\n },\n {\n source: \"/reference/clients/js/literals\",\n destination: \"/reference/using/js/querybuilder#types-and-literals\",\n permanent: false,\n },\n {\n source: \"/reference/clients/js/objects\",\n destination: \"/reference/using/js/querybuilder#objects-and-paths\",\n permanent: false,\n },\n {\n source: \"/reference/clients/js/parameters\",\n destination: \"/reference/using/js/querybuilder#parameters\",\n permanent: false,\n },\n {\n source: \"/reference/clients/js/select\",\n destination: \"/reference/using/js/querybuilder#select\",\n permanent: false,\n },\n {\n source: \"/reference/clients/js/types\",\n destination: \"/reference/using/js/querybuilder#types-and-literals\",\n permanent: false,\n },\n {\n source: \"/reference/clients/js/update\",\n destination: \"/reference/using/js/querybuilder#update\",\n permanent: false,\n },\n {\n source: \"/reference/clients/js/with\",\n destination: \"/reference/using/js/querybuilder#with-blocks\",\n permanent: false,\n },\n {\n source: \"/reference/clients/js/reference\",\n destination: \"/reference/using/js/client#client-reference\",\n permanent: false,\n },\n {\n source: \"/reference/reference/connection\",\n destination: \"/reference/using/connection\",\n permanent: false,\n },\n {\n source: \"/reference/reference/dsn\",\n destination: \"/reference/using/connection#dsn\",\n permanent: false,\n },\n {\n source: \"/reference/clients/python/api/asyncio_client\",\n destination: \"/reference/using/python/client#asyncio-client\",\n permanent: false,\n },\n {\n source: \"/reference/clients/python/api/blocking_client\",\n destination: \"/reference/using/python/client#blocking-client\",\n permanent: false,\n },\n {\n source: \"/reference/clients/python/installation\",\n destination: \"/reference/using/python#installation\",\n permanent: false,\n },\n {\n source: \"/reference/clients/python/usage\",\n destination: \"/reference/using/python#basic-usage\",\n permanent: false,\n },\n {\n source: \"/reference/clients/http/health-checks\",\n destination: \"/reference/running/http#health-checks\",\n permanent: false,\n },\n {\n source: \"/reference/clients/http/protocol\",\n destination: \"/reference/using/http\",\n permanent: false,\n },\n {\n source: \"/reference/clients/:path*\",\n destination: \"/reference/using/:path*\",\n permanent: false,\n },\n {\n source: \"/reference/reference/configuration\",\n destination: \"/reference/running/configuration\",\n permanent: false,\n },\n {\n source: \"/reference/reference/environment\",\n destination: \"/reference/running/configuration#environment-variables\",\n permanent: false,\n },\n {\n source: \"/reference/reference/gel_toml\",\n destination: \"/reference/using/projects#gel-toml\",\n permanent: false,\n },\n {\n source: \"/reference/reference/http\",\n destination: \"/reference/running/http\",\n permanent: false,\n },\n {\n source: \"/reference/reference/projects\",\n destination: \"/reference/using/projects\",\n permanent: false,\n },\n {\n source: \"/reference/reference/protocol/:path*\",\n destination: \"/resources/protocol/:path*\",\n permanent: false,\n },\n {\n source: \"/reference/reference/admin/databases\",\n destination: \"/reference/datamodel/branches\",\n permanent: false,\n },\n {\n source: \"/reference/reference/admin/:path*\",\n destination: \"/reference/running/admin/:path*\",\n permanent: false,\n },\n {\n source: \"/reference/reference/backend-ha\",\n destination: \"/reference/running/backend-ha\",\n permanent: false,\n },\n {\n source: \"/resources/guides/deployment/:path*\",\n destination: \"/reference/running/deployment/:path*\",\n permanent: false,\n },\n {\n source: \"/reference/reference/postgis\",\n destination: \"/reference/stdlib/postgis\",\n permanent: true,\n },\n {\n source: \"/reference/cli/:path*\",\n destination: \"/reference/using/cli/:path*\",\n permanent: false,\n },\n {\n source: \"/cli/:path*\",\n destination: \"/reference/using/cli/:path*\",\n permanent: false,\n },\n];\n" }, { "path": "docs/reference/ai/extai.rst", "content": ".. _ref_ai_extai_reference:\n\n=======\next::ai\n=======\n\nThis reference documents the |Gel| ``ext::ai`` extension components, configuration options, and database APIs.\n\n\nEnabling the Extension\n======================\n\nThe AI extension can be enabled using the :ref:`extension ` mechanism:\n\n.. code-block:: sdl\n\n using extension ai;\n\nConfiguration\n=============\n\nThe AI extension can be configured using ``configure session`` or ``configure current branch``:\n\n.. code-block:: edgeql\n\n configure current branch\n set ext::ai::Config::indexer_naptime := 'PT30S';\n\nConfiguration Properties\n------------------------\n\n* ``indexer_naptime``: Duration\n Specifies minimum delay between deferred ``ext::ai::index`` indexer runs.\n\nView current configuration:\n\n.. code-block:: edgeql\n\n select cfg::Config.extensions[is ext::ai::Config]{*};\n\nReset configuration:\n\n.. code-block:: edgeql\n\n configure current branch\n reset ext::ai::Config::indexer_naptime;\n\n\n.. _ref_ai_extai_reference_ui:\n\nUI\n==\n\nThe AI section of the UI can be accessed via the sidebar after the extension\nhas been enabled in the schema. It provides ways to manage provider\nconfigurations and RAG prompts, as well as try out different settings in the\nplayground.\n\nPlayground tab\n--------------\n\nProvides an interactive environment for testing and configuring the built-in\nRAG.\n\n.. image:: images/ui_playground.png\n :alt: Screenshot of the Playground tab of the UI depicting an empty message window and three input fields set with default values.\n :width: 100%\n\nComponents:\n\n* Message window: Displays conversation history between the user and the LLM.\n* Model: Dropdown menu for selecting the text generation model.\n* Prompt: Dropdown menu for selecting the RAG prompt template.\n* Context Query: Input field for entering an EdgeQL expression returning a set of objects with AI indexes.\n\n\nPrompts tab\n-----------\n\nProvides ways to manage system prompts used in the built-in RAG.\n\n.. image:: images/ui_prompts.png\n :alt: Screenshot of the Prompts tab of the UI depicting an expanded prompt configuration menu.\n :width: 100%\n\nProviders tab\n-------------\n\nEnables management of API configurations for AI API providers.\n\n.. image:: images/ui_providers.png\n :alt: Screenshot of the Providers tab of the UI depicting an expanded provider configuration menu.\n :width: 100%\n\n\n\n.. _ref_ai_extai_reference_index:\n\nIndex\n=====\n\nThe ``ext::ai::index`` creates a deferred semantic similarity index of an\nexpression on a type.\n\n.. code-block:: sdl-diff\n\n module default {\n type Astronomy {\n content: str;\n + deferred index ext::ai::index(embedding_model := 'text-embedding-3-small')\n + on (.content);\n }\n };\n\n\nParameters:\n\n* ``embedding_model``- The name of the model to use for embedding generation as\n a string.\n* ``distance_function``- The function to use for determining semantic\n similarity. Default: ``ext::ai::DistanceFunction.Cosine``\n* ``index_type``- The type of index to create. Currently the only option is the\n default: ``ext::ai::IndexType.HNSW``.\n* ``index_parameters``- A named tuple of additional index parameters:\n\n * ``m``- The maximum number of edges of each node in the graph. Increasing\n can increase the accuracy of searches at the cost of index size. Default:\n ``32``\n * ``ef_construction``- Dictates the depth and width of the search when\n building the index. Higher values can lead to better connections and more\n accurate results at the cost of time and resource usage when building the\n index. Default: ``100``\n\n* ``dimensions``: int64 (Optional) - Embedding dimensions\n* ``truncate_to_max``: bool (Default: False)\n\n\nFunctions\n=========\n\n.. list-table::\n :class: funcoptable\n\n * - :eql:func:`ext::ai::to_context`\n - :eql:func-desc:`ext::ai::to_context`\n\n * - :eql:func:`ext::ai::search`\n - :eql:func-desc:`ext::ai::search`\n\n\n------------\n\n\n.. eql:function:: ext::ai::to_context(object: anyobject) -> str\n\n Returns the indexed expression value for an object with an ``ext::ai::index``.\n\n **Example**:\n\n Schema:\n\n .. code-block:: sdl\n\n module default {\n type Astronomy {\n topic: str;\n content: str;\n deferred index ext::ai::index(embedding_model := 'text-embedding-3-small')\n on (.topic ++ ' ' ++ .content);\n }\n };\n\n Data:\n\n .. code-block:: edgeql-repl\n\n db> insert Astronomy {\n ... topic := 'Mars',\n ... content := 'Skies on Mars are red.'\n ... }\n db> insert Astronomy {\n ... topic := 'Earth',\n ... content := 'Skies on Earth are blue.'\n ... }\n\n Results of calling ``to_context``:\n\n .. code-block:: edgeql-repl\n\n db> select ext::ai::to_context(Astronomy);\n\n {'Mars Skies on Mars are red.', 'Earth Skies on Earth are blue.'}\n\n\n------------\n\n\n.. eql:function:: ext::ai::search( \\\n object: anyobject, \\\n query: array \\\n ) -> optional tuple\n ext::ai::search( \\\n object: anyobject, \\\n query: str \\\n ) -> optional tuple\n\n Searches objects using their :ref:`ai::index\n `.\n\n Returns tuples of (object, distance).\n\n .. versionadded:: 7.0\n\n If the ``query`` is a ``str``, the ai extension will make an embedding\n request to the provider and use the result to compute distances.\n\n To prevent unwanted provider calls, this functionality may only be\n used by roles with the :eql:permission:`ext::ai::perm::provider_call`\n permission.\n\n .. code-block:: edgeql-repl\n\n db> with query := >$query\n ... select ext::ai::search(Knowledge, query);\n\n {\n (\n object := default::Knowledge {id: 9af0d0e8-0880-11ef-9b6b-4335855251c4},\n distance := 0.20410746335983276\n ),\n (\n object := default::Knowledge {id: eeacf638-07f6-11ef-b9e9-57078acfce39},\n distance := 0.7843298847773637\n ),\n (\n object := default::Knowledge {id: f70863c6-07f6-11ef-b9e9-3708318e69ee},\n distance := 0.8560434728860855\n ),\n }\n\n\nScalar and Object Types\n=======================\n\nProvider Configuration Types\n----------------------------\n\n.. list-table::\n :class: funcoptable\n\n * - :eql:type:`ext::ai::ProviderAPIStyle`\n - Enum defining supported API styles\n\n * - :eql:type:`ext::ai::ProviderConfig`\n - Abstract base configuration for AI providers.\n\n\nProvider configurations are required for AI indexes and RAG functionality.\n\nExample provider configuration:\n\n.. code-block:: edgeql\n\n configure current database\n insert ext::ai::OpenAIProviderConfig {\n secret := 'sk-....',\n };\n\n.. note::\n\n All provider types require the ``secret`` property be set with a string\n containing the secret provided by the AI vendor.\n\n\n.. note::\n\n ``ext::ai::CustomProviderConfig requires an ``api_style`` property be set.\n\n\n---------\n\n\n.. eql:type:: ext::ai::ProviderAPIStyle\n\n Enum defining supported API styles:\n\n * ``OpenAI``\n * ``Anthropic``\n\n\n---------\n\n\n.. eql:type:: ext::ai::ProviderConfig\n\n Abstract base configuration for AI providers.\n\n Properties:\n\n * ``name``: str (Required) - Unique provider identifier\n * ``display_name``: str (Required) - Human-readable name\n * ``api_url``: str (Required) - Provider API endpoint\n * ``client_id``: str (Optional) - Provider-supplied client ID\n * ``secret``: str (Required) - Provider API secret\n * ``api_style``: ProviderAPIStyle (Required) - Provider's API style\n\n Provider-specific types:\n\n * ``ext::ai::OpenAIProviderConfig``\n * ``ext::ai::MistralProviderConfig``\n * ``ext::ai::AnthropicProviderConfig``\n * ``ext::ai::CustomProviderConfig``\n\n Each inherits from :eql:type:`ext::ai::ProviderConfig` with provider-specific defaults.\n\n\nModel Types\n-----------\n\n.. list-table::\n :class: funcoptable\n\n * - :eql:type:`ext::ai::Model`\n - Abstract base type for AI models.\n\n * - :eql:type:`ext::ai::EmbeddingModel`\n - Abstract type for embedding models.\n\n * - :eql:type:`ext::ai::TextGenerationModel`\n - Abstract type for text generation models.\n\n\n.. _ref_ai_extai_reference_embedding_models:\n\nEmbedding models\n^^^^^^^^^^^^^^^^\n\nOpenAI (`documentation `__)\n\n* ``text-embedding-3-small``\n* ``text-embedding-3-large``\n* ``text-embedding-ada-002``\n\nMistral (`documentation `__)\n\n* ``mistral-embed``\n\nOllama (`documentation `__)\n\n* ``nomic-embed-text``\n* ``bge-m3``\n\n\n.. _ref_ai_extai_reference_text_generation_models:\n\nText generation models\n^^^^^^^^^^^^^^^^^^^^^^\n\nOpenAI (`documentation `__)\n\n* ``gpt-3.5-turbo``\n* ``gpt-4-turbo-preview``\n\nMistral (`documentation `__)\n\n* ``mistral-small-latest``\n* ``mistral-medium-latest``\n* ``mistral-large-latest``\n\nAnthropic (`documentation `__)\n\n* ``claude-3-haiku-20240307``\n* ``claude-3-sonnet-20240229``\n* ``claude-3-opus-20240229``\n\nOllama (`documentation `__)\n\n* ``llama3.2``\n* ``llama3.3``\n\nWhen using RAG, It is possible to specify a text generation model using a URI,\ncombining the provider name (in lower case), and the model name.\n\n- eg. ``\"openai:gpt-5\"``\n- eg. ``\"anthropic:claude-opus-4-20250514\"``\n\nUsing this form allows text generation from models which are not explicitly\ninstantiated as a :eql:type:`ext::ai::TextGenerationModel`\n\n\n---------\n\n.. eql:type:: ext::ai::Model\n\n Abstract base type for AI models.\n\n Annotations:\n\n * ``model_name`` - Model identifier\n * ``model_provider`` - Provider identifier\n\n---------\n\n.. eql:type:: ext::ai::EmbeddingModel\n\n Abstract type for embedding models.\n\n Annotations:\n\n * ``embedding_model_max_input_tokens`` - Maximum tokens per input\n * ``embedding_model_max_batch_tokens`` - Maximum tokens per batch. Default: ``'8191'``.\n * ``embedding_model_max_batch_size`` - Maximum inputs per batch. Optional.\n * ``embedding_model_max_output_dimensions`` - Maximum embedding dimensions\n * ``embedding_model_supports_shortening`` - Input shortening support flag\n\n---------\n\n.. eql:type:: ext::ai::TextGenerationModel\n\n Abstract type for text generation models.\n\n Annotations:\n\n * ``text_gen_model_context_window`` - Model's context window size\n\n\nIndexing Types\n--------------\n\n.. list-table::\n :class: funcoptable\n\n * - :eql:type:`ext::ai::DistanceFunction`\n - Enum for similarity metrics.\n\n * - :eql:type:`ext::ai::IndexType`\n - Enum for index implementations.\n\n---------\n\n.. eql:type:: ext::ai::DistanceFunction\n\n Enum for similarity metrics.\n\n * ``Cosine``\n * ``InnerProduct``\n * ``L2``\n\n---------\n\n.. eql:type:: ext::ai::IndexType\n\n Enum for index implementations.\n\n * ``HNSW``\n\n\n\nPrompt Types\n------------\n\n.. list-table::\n :class: funcoptable\n\n * - :eql:type:`ext::ai::ChatParticipantRole`\n - Enum for chat roles.\n\n * - :eql:type:`ext::ai::ChatPromptMessage`\n - Type for chat prompt messages.\n\n * - :eql:type:`ext::ai::ChatPrompt`\n - Type for chat prompt configuration.\n\nExample custom prompt configuration:\n\n.. code-block:: edgeql\n\n insert ext::ai::ChatPrompt {\n name := 'test-prompt',\n messages := (\n insert ext::ai::ChatPromptMessage {\n participant_role := ext::ai::ChatParticipantRole.System,\n content := \"Your message content\"\n }\n )\n };\n\n\n---------\n\n.. eql:type:: ext::ai::ChatParticipantRole\n\n Enum for chat roles.\n\n * ``System``\n * ``User``\n * ``Assistant``\n * ``Tool``\n\n---------\n\n.. eql:type:: ext::ai::ChatPromptMessage\n\n Type for chat prompt messages.\n\n Properties:\n\n * ``participant_role``: ChatParticipantRole (Required)\n * ``participant_name``: str (Optional)\n * ``content``: str (Required)\n\n---------\n\n.. eql:type:: ext::ai::ChatPrompt\n\n Type for chat prompt configuration.\n\n Properties:\n\n * ``name``: str (Required)\n * ``messages``: set of ChatPromptMessage (Required)\n\n\nPermissions\n===========\n\n.. _ref_ai_extai_reference_permissions:\n\n.. versionadded:: 7.0\n\n.. list-table::\n :class: funcoptable\n\n * - :eql:permission:`ext::ai::perm::provider_call`\n * - :eql:permission:`ext::ai::perm::chat_prompt_read`\n * - :eql:permission:`ext::ai::perm::chat_prompt_write`\n\n---------\n\n.. eql:permission:: ext::ai::perm::provider_call\n\n Gives permission to make ai provider calls.\n\n Required to call :eql:func:`ext::ai::search` using text directly\n instead of an already generated embedding.\n\n---------\n\n.. eql:permission:: ext::ai::perm::chat_prompt_read\n\n Gives permission to read chat prompt configuration.\n\n---------\n\n.. eql:permission:: ext::ai::perm::chat_prompt_write\n\n Gives permission to modify chat prompt configuration.\n" }, { "path": "docs/reference/ai/extvectorstore.rst", "content": ":orphan:\n\n.. _ref_extvectorstore_reference:\n\n================\next::vectorstore\n================\n\n\nThe ``ext::vectorstore`` extension package provides simplified vectorstore\nworkflows for |Gel|, built on top of the pgvector integration. It includes\npredefined vector dimensions and a base schema for vector storage records.\n\n\nEnabling the extension\n======================\n\nThe extension package can be installed using the :gelcmd:`extension` CLI\ncommand:\n\n.. code-block:: bash\n\n $ gel extension install vectorstore\n\n\nIt can be enabled using the :ref:`extension `\nmechanism:\n\n.. code-block:: sdl\n\n using extension vectorstore;\n\n\nThe Vectorstore extension is designed to be used in combination with the\n:ref:`Vectostore Python binding ` or other\nintegrations, rather than on its own.\n\n\nTypes\n=====\n\nVector Types\n------------\n\nThe extension provides two pre-defined vector types with different dimensions:\n\n- ``ext::vectorstore::vector_1024``: 1024-dimensional vector\n- ``ext::vectorstore::vector_1536``: 1536-dimensional vector\n\nAll vector types extend ``ext::pgvector::vector`` with their respective dimensions.\n\n\nRecord Types\n------------\n\n.. eql:type:: ext::vectorstore::BaseRecord\n\n Abstract type that defines the basic structure for vector storage records.\n\n Properties:\n\n * ``collection: str`` (required): Identifies the collection the record belongs to\n * ``text: str``: Associated text content\n * ``embedding: ext::pgvector::vector``: The vector embedding\n * ``external_id: str``: External identifier with unique constraint\n * ``metadata: json``: Additional metadata in JSON format\n\n\n.. eql:type:: ext::vectorstore::DefaultRecord\n\n Extends :eql:type:`ext::vectorstore::BaseRecord` with specific\n configurations.\n\n Properties:\n\n * Inherits all properties from :eql:type:`ext::vectorstore::BaseRecord`\n * Specializes ``embedding`` to use ``vector_1536`` type\n * Includes an HNSW cosine similarity index on the embedding with:\n\n * ``m = 16``\n * ``ef_construction = 128``\n\n\n" }, { "path": "docs/reference/ai/http.rst", "content": ".. _ref_ai_http_reference:\n\n========\nHTTP API\n========\n\n.. note::\n\n All |Gel| server HTTP endpoints require :ref:`authentication\n `, such as `HTTP Basic Authentication\n `_\n with Gel username and password.\n\n\nEmbeddings\n==========\n\n``POST``: ``https://:/branch//ai/embeddings``\n\nGenerates text embeddings using the specified embeddings model.\n\n\nRequest headers\n---------------\n\n* ``Content-Type: application/json`` (required)\n\n\nRequest body\n------------\n\n* ``inputs`` (array of strings, or single string, required): The text items to use as the basis\n for embeddings generation.\n* ``model`` (string, required): The name of the embedding model to use. You may\n use any of the supported :ref:`embedding models\n `.\n* ``dimensions`` (number, optional): The number of dimensions to truncate to.\n\n* ``user`` (string, optional): A user identifier for the request.\n\n\nExample request\n---------------\n\n.. code-block:: bash\n\n $ curl --user : --json '{\\\n \"inputs\": [\"What color is the sky on Mars?\"],\\\n \"model\": \"text-embedding-3-small\"\\\n }' http://localhost:10931/branch/main/ai/embeddings\n\n\nResponse\n--------\n\n* **HTTP status**: 200 OK\n* **Content-Type**: application/json\n* **Body**:\n\n\n.. code-block:: json\n\n {\n \"object\": \"list\",\n \"data\": [\n {\n \"object\": \"embedding\",\n \"index\": 0,\n \"embedding\": [-0.009434271, 0.009137661]\n }\n ],\n \"model\": \"text-embedding-3-small\",\n \"usage\": {\n \"prompt_tokens\": 8,\n \"total_tokens\": 8\n }\n }\n\n.. note::\n\n The ``embedding`` property is shown here with only two values for brevity,\n but an actual response would contain many more values.\n\n\nError response\n--------------\n\n* **HTTP status**: 400 Bad Request\n* **Content-Type**: application/json\n* **Body**:\n\n .. code-block:: json\n\n {\n \"message\": \"missing or empty required \\\"model\\\" value in request\",\n \"type\": \"BadRequestError\"\n }\n\nRAG\n===\n\n``POST``: ``https://:/branch//ai/rag``\n\nPerforms retrieval-augmented text generation using the specified model based on\nthe provided text query and the database content selected using similarity\nsearch.\n\n\nRequest headers\n---------------\n\n* ``Content-Type: application/json`` (required)\n\n\nRequest body\n------------\n\n* ``context`` (object, required): Settings that define the context of the query.\n * ``query`` (string, required): Specifies an expression to determine the relevant objects and index to serve as context for text generation. You may set this to any expression that produces a set of objects, even if it is not a standalone query.\n * ``variables`` (object, optional): A dictionary of variables for use in the context query.\n * ``globals`` (object, optional): A dictionary of globals for use in the context query.\n * ``max_object_count`` (number, optional): Maximum number of objects to retrieve; default is 5.\n\n* ``model`` (string, required): The name of the text generation model to use. It is possible to specify the\n model name as a URI, eg. ``openai:gpt-5``. See: :ref:`text generation models\n `.\n\n* ``query`` (string, required): The query string used as the basis for text generation.\n\n* ``stream`` (boolean, optional): Specifies whether the response should be streamed. Defaults to false.\n\n* ``prompt`` (object, optional): Settings that define a prompt. Omit to use the default prompt.\n * ``name`` (string, optional): Name of predefined prompt.\n * ``id`` (string, optional): ID of predefined prompt.\n * ``custom`` (array of objects, optional): Custom prompt messages, each containing a ``role`` and ``content``. If no ``name`` or ``id`` was provided, the custom messages provided here become the prompt. If one of those was provided, these messages will be added to that existing prompt.\n * ``role`` (string): \"system\", \"user\", \"assistant\", or \"tool\".\n * ``content`` (string | array): Content of the message.\n * For ``role: \"system\"``: Must be a string.\n * For ``role: \"user\"``: Must be an array of content blocks, e.g., ``[{\"type\": \"text\", \"text\": \"...\"}]``.\n * For ``role: \"assistant\"``: Must be a string (the assistant's text response). May optionally include ``tool_calls``.\n * For ``role: \"tool\"``: Must be a string (the result of the tool call). Requires ``tool_call_id``.\n * ``tool_call_id`` (string, optional): Identifier for the tool call whose result this message represents (required if ``role: \"tool\"``).\n * ``tool_calls`` (array, optional): Array of tool calls requested by the assistant (used if ``role: \"assistant\"``). Each object should follow the format: ``{\"id\": \"...\", \"type\": \"function\", \"function\": {\"name\": \"...\", \"arguments\": \"...\"}}``. Arguments should be a JSON string.\n\n* ``temperature`` (number, optional): Sampling temperature.\n\n* ``top_p`` (number, optional): Nucleus sampling parameter.\n\n* ``max_tokens`` (number, optional): Maximum tokens to generate.\n\n* ``seed`` (number, optional): Random seed.\n\n* ``safe_prompt`` (boolean, optional): Enable safety features.\n\n* ``top_k`` (number, optional): Top-k sampling parameter.\n\n* ``logit_bias`` (object, optional): Token biasing.\n\n* ``logprobs`` (number, optional): Return token log probabilities.\n\n* ``user`` (string, optional): User identifier.\n\n* ``tools`` (array, optional): A list of tools the model may call. Each tool\n has a ``type`` (\"function\") and a ``function`` object with ``name``,\n ``description`` (optional), and ``parameters`` (JSON schema). Example:\n ``[{\"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"description\": \"Get the current weather\", \"parameters\": {\"type\": \"object\", \"properties\": {\"location\": {\"type\": \"string\"}}, \"required\": [\"location\"]}}}]``\n\n\nExample request\n---------------\n\n.. code-block:: bash\n\n $ curl --user : --json '{\\\n \"query\": \"What color is the sky on Mars?\",\\\n \"model\": \"gpt-4-turbo-preview\",\\\n \"context\": {\"query\":\"Knowledge\"}\\\n }' http://:/branch/main/ai/rag\n\n\nResponse\n--------\n\n* **HTTP status**: 200 OK\n* **Content-Type**: application/json\n* **Body**: A JSON object containing the RAG response details.\n\n .. code-block:: json\n\n {\n \"id\": \"chatcmpl-xxxxxxxxxxxxxxxxxxxxxxxxxxxxx\",\n \"model\": \"gpt-4-turbo-preview\",\n \"text\": \"The sky on Mars typically appears butterscotch or reddish due to the fine dust particles suspended in the atmosphere.\",\n \"finish_reason\": \"stop\",\n \"usage\": {\n \"prompt_tokens\": 50,\n \"completion_tokens\": 30,\n \"total_tokens\": 80\n },\n \"logprobs\": null,\n \"tool_calls\": null\n }\n\n * ``id`` (string): Unique identifier for the chat completion.\n * ``model`` (string): The model used for the chat completion.\n * ``text`` (string | null): The main text content of the response message.\n * ``finish_reason`` (string | null): The reason the model stopped generating tokens (e.g., \"stop\", \"length\", \"tool_calls\").\n * ``usage`` (object | null): Token usage statistics for the request.\n * ``logprobs`` (object | null): Log probability information for the generated tokens (if requested).\n * ``tool_calls`` (array | null): Any tool calls requested by the model. Each element contains ``id``, ``type`` (\"function\"), ``name``, and ``args`` (parsed JSON object).\n\n\nError response\n--------------\n\n* **HTTP status**: 400 Bad Request\n* **Content-Type**: application/json\n* **Body**:\n\n .. code-block:: json\n\n {\n \"message\": \"missing required 'query' in request 'context' object\",\n \"type\": \"BadRequestError\"\n }\n\n\nStreaming response (SSE)\n------------------------\n\nWhen the ``stream`` parameter is set to ``true``, the server uses `Server-Sent\nEvents\n`__\n(SSE) to stream responses. Here is a detailed breakdown of the typical\nsequence and structure of events in a streaming response:\n\n* **HTTP Status**: 200 OK\n* **Content-Type**: text/event-stream\n* **Cache-Control**: no-cache\n\nThe stream consists of a sequence of five events, each encapsulating part of\nthe response in a structured format:\n\n1. **Message start**\n\n * Event type: ``message_start``\n\n * Data: Starts a message, specifying identifiers, roles, and initial usage.\n\n .. code-block:: json\n\n {\n \"type\": \"message_start\",\n \"message\": {\n \"id\": \"\",\n \"role\": \"assistant\",\n \"model\": \"\",\n \"usage\": { \"prompt_tokens\": 10 }\n }\n }\n\n2. **Content block start**\n\n * Event type: ``content_block_start``\n\n * Data: Marks the beginning of a new content block (either text or a tool call).\n\n .. code-block:: json\n\n {\n \"type\": \"content_block_start\",\n \"index\": 0,\n \"content_block\": {\n \"type\": \"text\",\n \"text\": \"\"\n }\n }\n\n Or for a tool call:\n\n .. code-block:: json\n\n {\n \"type\": \"content_block_start\",\n \"index\": 0,\n \"content_block\": {\n \"id\": \"\",\n \"type\": \"tool_use\",\n \"name\": \"\",\n \"args\": \"{...\"\n }\n }\n\n\n3. **Content block delta**\n\n * Event type: ``content_block_delta``\n\n * Data: Incrementally updates the content, appending more text or tool arguments. Includes logprobs if requested.\n\n .. code-block:: json\n\n {\n \"type\": \"content_block_delta\",\n \"index\": 0,\n \"delta\": {\n \"type\": \"text_delta\",\n \"text\": \"The\"\n },\n \"logprobs\": null\n }\n\n Or for tool arguments:\n\n .. code-block:: json\n\n {\n \"type\": \"content_block_delta\",\n \"index\": 0,\n \"delta\": {\n \"type\": \"tool_call_delta\",\n \"args\": \"{\\\"location\"\n }\n }\n\n Subsequent ``content_block_delta`` events add more text/arguments to the message.\n\n4. **Content block stop**\n\n * Event type: ``content_block_stop``\n\n * Data: Marks the end of a content block.\n\n .. code-block:: json\n\n {\n \"type\": \"content_block_stop\",\n \"index\": 0\n }\n\n5. **Message delta**\n\n * Event type: ``message_delta``\n\n * Data: Provides final message-level updates like the stop reason and final usage statistics.\n\n .. code-block:: json\n\n {\n \"type\": \"message_delta\",\n \"delta\": {\n \"stop_reason\": \"stop\"\n },\n \"usage\": { \"prompt_tokens\": 10 }\n }\n\n\n6. **Message stop**\n\n * Event type: ``message_stop``\n\n * Data: Marks the end of the message.\n\n .. code-block:: json\n\n {\"type\": \"message_stop\"}\n\nEach event is sent as a separate SSE message, formatted as shown above. The\nconnection is closed after all events are sent, signaling the end of the\nstream.\n\n**Example SSE response**\n\n.. code-block:: text\n :class: collapsible\n\n event: message_start\n data: {\"type\": \"message_start\", \"message\": {\"id\": \"chatcmpl-9MzuQiF0SxUjFLRjIdT3mTVaMWwiv\", \"role\": \"assistant\", \"model\": \"gpt-4-0125-preview\", \"usage\": {\"prompt_tokens\": 10}}}\n\n event: content_block_start\n data: {\"type\": \"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\n event: content_block_delta\n data: {\"type\": \"content_block_delta\",\"index\":0,\"delta\":{\"type\": \"text_delta\", \"text\": \"The\"}, \"logprobs\": null}\n\n event: content_block_delta\n data: {\"type\": \"content_block_delta\",\"index\":0,\"delta\":{\"type\": \"text_delta\", \"text\": \" skies\"}, \"logprobs\": null}\n\n event: content_block_delta\n data: {\"type\": \"content_block_delta\",\"index\":0,\"delta\":{\"type\": \"text_delta\", \"text\": \" on\"}, \"logprobs\": null}\n\n event: content_block_delta\n data: {\"type\": \"content_block_delta\",\"index\":0,\"delta\":{\"type\": \"text_delta\", \"text\": \" Mars\"}, \"logprobs\": null}\n\n event: content_block_delta\n data: {\"type\": \"content_block_delta\",\"index\":0,\"delta\":{\"type\": \"text_delta\", \"text\": \" are\"}, \"logprobs\": null}\n\n event: content_block_delta\n data: {\"type\": \"content_block_delta\",\"index\":0,\"delta\":{\"type\": \"text_delta\", \"text\": \" red\"}, \"logprobs\": null}\n\n event: content_block_delta\n data: {\"type\": \"content_block_delta\",\"index\":0,\"delta\":{\"type\": \"text_delta\", \"text\": \".\"}, \"logprobs\": null}\n\n event: content_block_stop\n data: {\"type\": \"content_block_stop\",\"index\":0}\n\n event: message_delta\n data: {\"type\": \"message_delta\", \"delta\": {\"stop_reason\": \"stop\"}, \"usage\": {\"completion_tokens\": 7, \"total_tokens\": 17}}\n\n event: message_stop\n data: {\"type\": \"message_stop\"}\n\n\n" }, { "path": "docs/reference/ai/index.rst", "content": ".. _ref_ai_overview:\n\n==\nAI\n==\n\n:edb-alt-title: Using Gel AI\n\n.. toctree::\n :hidden:\n :maxdepth: 3\n\n extai\n http\n python\n javascript\n\n|Gel| AI is a set of tools designed to enable you to ship AI-enabled apps with\npractically no effort.\n\n1. ``ext::ai``: this Gel extension automatically generates embeddings for your\n data. Works with OpenAI, Mistral AI, Anthropic, and any other provider with a\n compatible API.\n\n2. Python library: ``gel.ai``. Access all Gel AI features straight from your\n Python application.\n\n3. JavaScript library: ``@gel/ai``. Access all Gel AI features right from your\n JavaScript backend application.\n\n..\n 2. ``ext::vectorstore``: this extension is designed to replicate workflows that\n might be familiar to you from vectorstore-style databases. Powered by\n ``pgvector``, it allows you to store and search for embedding vectors, and\n integrates with popular AI frameworks.\n\n" }, { "path": "docs/reference/ai/javascript.rst", "content": ".. _ref_ai_javascript_reference:\n\n==============\nJavaScript API\n==============\n\n``@gel/ai`` is a wrapper around the :ref:`AI extension\n` in |Gel|.\n\n.. tabs::\n\n .. code-tab:: bash\n :caption: npm\n\n $ npm install @gel/ai\n\n .. code-tab:: bash\n :caption: yarn\n\n $ yarn add @gel/ai\n\n .. code-tab:: bash\n :caption: pnpm\n\n $ pnpm add @gel/ai\n\n .. code-tab:: bash\n :caption: bun\n\n $ bun add @gel/ai\n\n\nOverview\n========\n\nThe AI package is built on top of the regular |Gel| client objects.\n\n**Example**:\n\n.. code-block:: typescript\n\n import { createClient } from \"gel\";\n import { createRAGClient } from \"@gel/ai\";\n\n\n const client = createClient();\n\n const gpt4Ai = createRAGClient(client, {\n model: \"gpt-4-turbo-preview\",\n });\n\n const astronomyAi = gpt4Ai.withContext({\n query: \"Astronomy\"\n });\n\n console.log(\n await astronomyAi.queryRag(\"What color is the sky on Mars?\")\n );\n\n\nFactory functions\n=================\n\n.. js:function:: createRAGClient( \\\n client: Client, \\\n options: Partial = {} \\\n ): RAGClient\n\n Creates an instance of ``RAGClient`` with the specified client and options.\n\n :param client:\n A |Gel| client instance.\n\n :param string options.model:\n Required. Specifies the AI model to use. This could be a version of GPT\n or any other model supported by |Gel| AI. It is possible to specify the\n model name as a URI, eg. ``openai:gpt-5``. See::ref:`text generation\n models`.\n\n :param options.prompt:\n Optional. Defines the input prompt for the AI model. The prompt can be\n a simple string, an ID referencing a stored prompt, or a custom prompt\n structure that includes roles and content for more complex\n interactions. The default is the built-in system prompt.\n\n\nCore classes\n============\n\n\n.. js:class:: RAGClient\n\n Instances of ``RAGClient`` offer methods for client configuration and utilizing RAG.\n\n :ivar client:\n An instance of |Gel| client.\n\n .. js:method:: withConfig(options: Partial): RAGClient\n\n Returns a new ``RAGClient`` instance with updated configuration options.\n\n :param string options.model:\n Required. Specifies the AI model to use. This could be a version of GPT\n or any other model supported by |Gel| AI.\n\n :param options.prompt:\n Optional. Defines the input prompt for the AI model. The prompt can be\n a simple string, an ID referencing a stored prompt, or a custom prompt\n structure that includes roles and content for more complex\n interactions. The default is the built-in system prompt.\n\n .. js:method:: withContext(context: Partial): RAGClient\n\n Returns a new ``RAGClient`` instance with an updated query context.\n\n :param string context.query:\n Required. Specifies an expression to determine the relevant objects and\n index to serve as context for text generation. You may set this to any\n expression that produces a set of objects, even if it is not a\n standalone query.\n :param string context.variables:\n Optional. Variable settings required for the context query.\n :param string context.globals:\n Optional. Variable settings required for the context query.\n :param number context.max_object_count:\n Optional. A maximum number of objects to return from the context query.\n\n .. js:method:: async queryRag( \\\n message: string, \\\n context: QueryContext = this.context \\\n ): Promise\n\n Sends a query with context to the configured AI model and returns the\n response as a string.\n\n :param string message:\n Required. The message to be sent to the text generation provider's API.\n :param string context.query:\n Required. Specifies an expression to determine the relevant objects and\n index to serve as context for text generation. You may set this to any\n expression that produces a set of objects, even if it is not a\n standalone query.\n :param string context.variables:\n Optional. Variable settings required for the context query.\n :param string context.globals:\n Optional. Variable settings required for the context query.\n :param number context.max_object_count:\n Optional. A maximum number of objects to return from the context query.\n\n .. js:method:: async streamRag( \\\n message: string, \\\n context: QueryContext = this.context \\\n ): AsyncIterable & PromiseLike\n\n Can be used in two ways:\n\n - as **an async iterator** - if you want to process streaming data in\n real-time as it arrives, ideal for handling long-running streams.\n\n - as **a Promise that resolves to a full Response object** - you have\n complete control over how you want to handle the stream, this might be\n useful when you want to manipulate the raw stream or parse it in a custom way.\n\n :param string message:\n Required. The message to be sent to the text generation provider's API.\n :param string context.query:\n Required. Specifies an expression to determine the relevant objects and\n index to serve as context for text generation. You may set this to any\n expression that produces a set of objects, even if it is not a\n standalone query.\n :param string context.variables:\n Optional. Variable settings required for the context query.\n :param string context.globals:\n Optional. Variable settings required for the context query.\n :param number context.max_object_count:\n Optional. A maximum number of objects to return from the context query.\n\n .. js:method:: async generateEmbeddings( \\\n inputs: string[], \\\n model: string \\\n ): Promise\n\n Generates embeddings for the array of strings.\n\n :param string[] inputs:\n Required. Strings array to generate embeddings for.\n :param string model:\n Required. Specifies the AI model to use.\n" }, { "path": "docs/reference/ai/python.rst", "content": ".. _ref_ai_python_reference:\n\n==========\nPython API\n==========\n\nThe ``gel.ai`` package is an optional binding of the :ref:`AI extension\n` in |Gel|.\n\n.. code-block:: bash\n\n $ pip install 'gel[ai]'\n\n\nBlocking and async API\n======================\n\nThe AI binding is built on top of the regular |Gel| client objects, providing\nboth blocking and asynchronous versions of its API.\n\n**Blocking client example**:\n\n.. code-block:: python\n\n import gel\n import gel.ai\n\n client = gel.create_client()\n gpt4ai = gel.ai.create_rag_client(\n client,\n model=\"gpt-4-turbo-preview\"\n )\n\n astronomy_ai = gpt4ai.with_context(\n query=\"Astronomy\"\n )\n\n print(\n astronomy_ai.query_rag(\"What color is the sky on Mars?\")\n );\n\n for data in astronomy_ai.stream_rag(\"What color is the sky on Mars?\"):\n print(data)\n\n\n**Async client example**:\n\n.. code-block:: python\n\n import gel\n import gel.ai\n import asyncio\n\n client = gel.create_async_client()\n\n async def main():\n gpt4ai = await gel.ai.create_async_rag_client(\n client,\n model=\"gpt-4-turbo-preview\"\n )\n astronomy_ai = gpt4ai.with_context(\n query=\"Astronomy\"\n )\n query = \"What color is the sky on Mars?\"\n print(\n await astronomy_ai.query_rag(query)\n );\n\n #or streamed\n async for data in blog_ai.stream_rag(query):\n print(data)\n\n asyncio.run(main())\n\n\nFactory functions\n=================\n\n.. py:function:: create_rag_client(client, **kwargs) -> RAGClient\n\n Creates an instance of ``RAGClient`` with the specified client and options.\n\n This function ensures that the client is connected before initializing the\n AI with the specified options.\n\n :param client:\n A |Gel| client instance.\n\n :param kwargs:\n Keyword arguments that are passed to the ``RAGOptions`` data class to\n configure AI-specific options. These options are:\n\n * ``model``: The name of the model to be used. (required)\n * ``prompt``: An optional prompt to guide the model's behavior.\n ``None`` will result in the client using the default prompt.\n (default: ``None``)\n\n.. py:function:: create_async_rag_client(client, **kwargs) -> AsyncRAGClient\n\n Creates an instance of ``AsyncRAGClient`` w/ the specified client & options.\n\n This function ensures that the client is connected asynchronously before\n initializing the AI with the specified options.\n\n :param client:\n An asynchronous |Gel| client instance.\n\n :param kwargs:\n Keyword arguments that are passed to the ``RAGOptions`` data class to\n configure AI-specific options. These options are:\n\n * ``model``: The name of the model to be used. It is possible to specify\n the model name as a URI, eg. ``openai:gpt-5``. See: :ref:`text\n generation models `.\n (required)\n * ``prompt``: An optional prompt to guide the model's behavior. (default: None)\n\n\nCore classes\n============\n\n.. py:class:: BaseRAGClient\n\n The base class for |Gel| AI clients.\n\n This class handles the initialization and configuration of AI clients and\n provides methods to modify their configuration and context dynamically.\n\n Both the blocking and async AI client classes inherit from this one, so\n these methods are available on an AI client of either type.\n\n :ivar options:\n An instance of :py:class:`RAGOptions`, storing the RAG options.\n\n :ivar context:\n An instance of :py:class:`QueryContext`, storing the context for AI\n queries.\n\n :ivar client_cls:\n A placeholder for the client class, should be implemented by subclasses.\n\n :param client:\n An instance of |Gel| client, which could be either a synchronous or\n asynchronous client.\n\n :param options:\n AI options to be used with the client.\n\n :param kwargs:\n Keyword arguments to initialize the query context.\n\n.. py:method:: with_config(**kwargs)\n\n Creates a new instance of the same class with modified configuration\n options. This method uses the current instance's configuration as a base and\n applies the changes specified in ``kwargs``.\n\n :param kwargs:\n Keyword arguments that specify the changes to the AI configuration.\n These changes are passed to the ``derive`` method of the current\n configuration options object. Possible keywords include:\n\n * ``model``: Specifies the AI model to be used. This must be a string.\n * ``prompt``: An optional prompt to guide the model's behavior. This is\n optional and defaults to None.\n\n.. py:method:: with_context(**kwargs)\n\n Creates a new instance of the same class with a modified context. This\n method preserves the current AI options and client settings, but uses the\n modified context specified by ``kwargs``.\n\n :param kwargs:\n Keyword arguments that specify the changes to the context. These changes\n are passed to the ``derive`` method of the current context object.\n Possible keywords include:\n\n * ``query``: The database query string.\n * ``variables``: A dictionary of variables used in the query.\n * ``globals``: A dictionary of global settings affecting the query.\n * ``max_object_count``: An optional integer to limit the number of\n objects returned by the query.\n\n\n.. py:class:: RAGClient\n\n A synchronous class for creating |Gel| AI clients.\n\n This class provides methods to send queries and receive responses using both\n blocking and streaming communication modes synchronously.\n\n :ivar client:\n An instance of ``httpx.AsyncClient`` used for making HTTP requests\n asynchronously.\n\n.. py:method:: query_rag(message, context=None) -> str\n\n Sends a request to the AI provider and returns the response as a string.\n\n This method uses a blocking HTTP POST request. It raises an HTTP exception\n if the request fails.\n\n :param message:\n The query string to be sent to the AI model.\n :param context:\n An optional ``QueryContext`` object to provide additional context for\n the query. If not provided, uses the default context of this AI client\n instance.\n\n.. py:method:: stream_rag(message, context=None)\n\n Opens a connection to the AI provider to stream query responses.\n\n This method yields data as it is received, utilizing Server-Sent Events\n (SSE) to handle streaming data. It raises an HTTP exception if the request\n fails.\n\n :param message:\n The query string to be sent to the AI model.\n :param context:\n An optional ``QueryContext`` object to provide additional context for\n the query. If not provided, uses the default context of this AI client\n instance.\n\n.. py:method:: generate_embeddings(*inputs: str, model: str) -> list[float]\n\n Generates embeddings for input texts.\n\n :param inputs:\n Input texts.\n :param model:\n The embedding model to use\n\n\n.. py:class:: AsyncRAGClient\n\n An asynchronous class for creating |Gel| AI clients.\n\n This class provides methods to send queries and receive responses using both\n blocking and streaming communication modes asynchronously.\n\n :ivar client:\n An instance of ``httpx.AsyncClient`` used for making HTTP requests\n asynchronously.\n\n.. py:method:: query_rag(message, context=None) -> str\n :noindex:\n\n Sends an async request to the AI provider, returns the response as a string.\n\n This method is asynchronous and should be awaited. It raises an HTTP\n exception if the request fails.\n\n :param message:\n The query string to be sent to the AI model.\n\n :param context:\n An optional ``QueryContext`` object to provide additional context for\n the query. If not provided, uses the default context of this AI client\n instance.\n\n.. py:method:: stream_rag(message, context=None)\n :noindex:\n\n Opens an async connection to the AI provider to stream query responses.\n\n This method yields data as it is received, using asynchronous Server-Sent\n Events (SSE) to handle streaming data. This is an asynchronous generator\n method and should be used in an async for loop. It raises an HTTP exception\n if the connection fails.\n\n :param message:\n The query string to be sent to the AI model.\n :param context:\n An optional ``QueryContext`` object to provide additional context for\n the query. If not provided, uses the default context of this AI client\n instance.\n\n.. py:method:: generate_embeddings(*inputs: str, model: str) -> list[float]\n :noindex:\n\n Generates embeddings for input texts.\n\n :param inputs:\n Input texts.\n :param model:\n The embedding model to use\n\n\nConfiguration classes\n=====================\n\n.. py:class:: ChatParticipantRole\n\n An enumeration of roles used when defining a custom text generation prompt.\n\n :cvar SYSTEM:\n Represents a system-level entity or process.\n :cvar USER:\n Represents a human user participating in the chat.\n :cvar ASSISTANT:\n Represents an AI assistant.\n :cvar TOOL:\n Represents a tool or utility used within the chat context.\n\n\n.. py:class:: Custom\n\n A single message in a custom text generation prompt.\n\n :ivar role:\n The role of the chat participant. Must be an instance of\n :py:class:`ChatParticipantRole`.\n :ivar content:\n The content associated with the role, expressed as a string.\n\n\n.. py:class:: Prompt\n\n The metadata and content of a text generation prompt.\n\n :ivar name:\n An optional name identifying the prompt.\n :ivar id:\n An optional unique identifier for the prompt.\n :ivar custom:\n An optional list of :py:class:`Custom` objects, each providing\n role-specific content within the prompt.\n\n\n.. py:class:: RAGOptions\n\n A data class for RAG options, specifying model and prompt settings.\n\n :ivar model:\n The name of the AI model.\n :ivar prompt:\n An optional :py:class:`Prompt` providing additional guiding information for\n the model.\n\n :method derive(kwargs):\n Creates a new instance of :py:class:`RAGOptions` by merging existing options\n with provided keyword arguments. Returns a new :py:class:`RAGOptions`\n instance with updated attributes.\n\n :param kwargs:\n Keyword arguments to update the current AI options. Possible\n keywords include:\n\n * ``model`` (str): Update the model name.\n * ``prompt`` (:py:class:`Prompt`): Update or set a new prompt object.\n\n\n.. py:class:: QueryContext\n\n A data class defining the context for a query to an AI model.\n\n :ivar query:\n The base query string.\n :ivar variables:\n An optional dictionary of variables used in the query.\n :ivar globals:\n An optional dictionary of global settings affecting the query.\n :ivar max_object_count:\n An optional integer specifying the maximum number of objects the query\n should return.\n\n :method derive(kwargs):\n Creates a new instance of :py:class:`QueryContext` by merging existing\n context with provided keyword arguments. Returns a new\n :py:class:`QueryContext` instance with updated attributes.\n\n :param kwargs:\n Keyword arguments to update the current query context. Possible\n keywords include:\n\n * ``query`` (str): Update the query string.\n * ``variables`` (dict): Update or set new variables for the query.\n * ``globals`` (dict): Update or set new global settings for the query.\n * ``max_object_count`` (int): Update the limit on the number of objects returned by the query.\n\n\n.. py:class:: RAGRequest\n\n A data class defining a request to a text generation model.\n\n :ivar model:\n The name of the AI model to query.\n :ivar prompt:\n An optional :py:class:`Prompt` associated with the request.\n :ivar context:\n The :py:class:`QueryContext` defining the query context.\n :ivar query:\n The specific query string to be sent to the model.\n :ivar stream:\n A boolean indicating whether the response should be streamed (True) or\n returned in a single response (False).\n\n :method to_httpx_request():\n Converts the RAGRequest into a dictionary suitable for making an HTTP\n request using the httpx library.\n\n" }, { "path": "docs/reference/ai/vectorstore_python.rst", "content": ":orphan:\n\n.. _ref_ai_vectorstore_python:\n\n======================\nVectorstore Python API\n======================\n\n\nCore Classes\n============\n\n.. py:class:: GelVectorstore\n\n A framework-agnostic interface for interacting with |Gel's| ext::vectorstore.\n\n This class provides methods for storing, retrieving, and searching\n vector embeddings. It follows vector database conventions and supports\n different embedding models.\n\n Args:\n\n * ``embedding_model`` (:py:class:`BaseEmbeddingModel`): The embedding model used to generate vectors.\n * ``collection_name`` (str): The name of the collection.\n * ``record_type`` (str): The schema type (table name) for storing records.\n * ``client_config`` (dict | None): The config for the |Gel| client.\n\n\n.. py:method:: add_items(self, items: list[InsertItem])\n\n Add multiple items to the vector store in a single transaction. Embeddings\n will be generated and stored for all items.\n\n Args:\n\n * ``items`` (list[:py:class:`InsertItem`]): List of items to add. Each contains:\n\n * ``text`` (str): The text content to be embedded\n * ``metadata`` (dict[str, Any]): Additional data to store\n\n Returns:\n\n * List of database record IDs for the inserted items.\n\n\n.. py:method:: add_vectors(self, records: list[InsertRecord])\n\n Add pre-computed vector embeddings to the store. Use this method when you have\n already generated embeddings and want to store them directly without re-computing them.\n\n Args:\n\n * ``records`` (list[:py:class:`InsertRecord`]): List of records. Each contains:\n\n * ``embedding`` (list[float]): Pre-computed embeddings\n * ``text`` (Optional[str]): Original text content\n * ``metadata`` (dict[str, Any]): Additional data to store\n\n Returns:\n\n * List of database record IDs for the inserted items.\n\n.. py:method:: delete(self, ids: list[uuid.UUID])\n\n Delete records from the vector store by their IDs.\n\n Args:\n\n * ``ids`` (list[uuid.UUID]): List of record IDs to delete.\n\n Returns:\n\n * List of deleted record IDs.\n\n.. py:method:: get_by_ids(self, ids: list[uuid.UUID]) -> list[Record]\n\n Retrieve specific records by their IDs.\n\n Args:\n\n * ``ids`` (list[uuid.UUID]): List of record IDs to retrieve.\n\n Returns:\n\n * List of retrieved records. Each result contains:\n\n * ``id`` (uuid.UUID): The record's unique identifier\n * ``text`` (Optional[str]): The original text content\n * ``embedding`` (Optional[list[float]]): The stored vector embedding\n * ``metadata`` (Optional[dict[str, Any]]): Any associated metadata\n\n.. py:method:: search_by_item(self, item: Any, filters: Optional[CompositeFilter] = None, limit: Optional[int] = 4) -> list[SearchResult]\n\n Search for similar items in the vector store.\n\n This method:\n\n 1. Generates an embedding for the input item\n 2. Finds records with similar embeddings\n 3. Optionally filters results based on metadata\n 4. Returns the most similar items up to the specified limit\n\n Args:\n\n * ``item`` (Any): The query item to find similar matches for. Must be compatible with the embedding model's target_type.\n * ``filters`` (Optional[:py:class:`CompositeFilter`]): Metadata-based filters to use.\n * ``limit`` (Optional[int]): Max number of results to return. Defaults to 4.\n\n Returns:\n\n * List of similar items, ordered by similarity. Each result contains:\n\n * ``id`` (uuid.UUID): The record's unique identifier\n * ``text`` (Optional[str]): The original text content\n * ``embedding`` (list[float]): The stored vector embedding\n * ``metadata`` (Optional[dict[str, Any]]): Any associated metadata\n * ``cosine_similarity`` (float): Similarity score (higher is more similar)\n\n.. py:method:: search_by_vector(self, vector: list[float], filter_expression: str = \"\", limit: Optional[int] = 4) -> list[SearchResult]\n\n Search using a pre-computed vector embedding. Useful when you have already computed\n the embedding or want to search with a modified/combined embedding vector.\n\n Args:\n\n * ``vector`` (list[float]): The query embedding to search with. Must match the dimensionality of stored embeddings.\n * ``filter_expression`` (str): Filter expression for metadata filtering.\n * ``limit`` (Optional[int]): Max number of results to return. Defaults to 4.\n\n Returns:\n\n * List of similar items, ordered by similarity. Each result contains:\n\n * ``id`` (uuid.UUID): The record's unique identifier\n * ``text`` (Optional[str]): The original text content\n * ``embedding`` (list[float]): The stored vector embedding\n * ``metadata`` (Optional[dict[str, Any]]): Any associated metadata\n * ``cosine_similarity`` (float): Similarity score (higher is more similar)\n\n.. py:method:: update_record(self, record: Record) -> Optional[uuid.UUID]\n\n Update an existing record in the vector store. Only specified fields will be updated.\n If text is provided but not embedding, a new embedding will be automatically generated.\n\n Args:\n\n * ``record`` (:py:class:`Record`):\n\n * ``id`` (uuid.UUID): The ID of the record to update\n * ``text`` (Optional[str]): New text content. If provided without embedding, a new embedding will be generated.\n * ``embedding`` (Optional[list[float]]): New vector embedding.\n * ``metadata`` (Optional[dict[str, Any]]): New metadata to store with the record. Completely replaces existing metadata.\n\n Returns:\n\n * The updated record's ID if found and updated, None if no record was found with the given ID.\n\n Raises:\n\n * ValueError: If no fields are specified for update.\n\n\n.. py:class:: BaseEmbeddingModel\n\n Abstract base class for embedding models.\n Any embedding model used with :py:class:`GelVectorstore` must implement this\n interface. The model is expected to convert input data (text, images, etc.)\n into a numerical vector representation.\n\n .. py:method:: __call__(self, item) -> list[float]\n\n Convert an input item into a list of floating-point values (vector\n embedding). Must be implemented in subclasses.\n\n Args:\n\n * ``item``: Input item to be converted to an embedding\n\n Returns:\n\n * list[float]: Vector embedding of the input item\n\n .. py:method:: dimensions(self) -> int\n\n Return the number of dimensions in the embedding vector.\n Must be implemented in subclasses.\n\n Returns:\n\n * int: Number of dimensions in the embedding vector\n\n .. py:method:: target_type(self) -> TypeVar\n\n Return the expected data type of the input (e.g., str for text, image\n for vision models). Must be implemented in subclasses.\n\n Returns:\n\n * TypeVar: Expected input data type\n\n\nData Classes\n============\n\n.. py:class:: InsertItem\n\n An item whose embedding will be created and stored alongside the item in the vector store.\n\n Args:\n\n * ``text`` (str): The text content to be embedded\n * ``metadata`` (dict[str, Any]): Additional data to store. Defaults to empty dict.\n\n.. py:class:: InsertRecord\n\n A record to be added to the vector store with embedding pre-computed.\n\n Args:\n\n * ``embedding`` (list[float]): Pre-computed embeddings\n * ``text`` (str | None): Original text content. Defaults to None.\n * ``metadata`` (dict[str, Any]): Additional data to store. Defaults to empty dict.\n\n.. py:class:: Record\n\n A record retrieved from the vector store, or an update record.\n Custom ``__init__`` so we can detect which fields the user passed\n (even if they pass None or {}).\n\n Args:\n\n * ``id`` (uuid.UUID): The record's unique identifier\n * ``text`` (str | None): The text content. Defaults to None.\n * ``embedding`` (list[float] | None): The vector embedding. Defaults to None.\n * ``metadata`` (dict[str, Any]): Additional data stored with the record. Defaults to empty dict.\n\n.. py:class:: SearchResult\n\n A search result from the vector store.\n\n Inherits from :py:class:`Record`\n\n Args:\n\n * ``cosine_similarity`` (float): Similarity score for the search result. Defaults to 0.0.\n\n\nMetadata Filtering\n==================\n\n.. py:class:: FilterOperator\n\n Enumeration of supported filter operators for metadata filtering.\n\n Values:\n\n * ``EQ``: Equal to (=)\n * ``NE``: Not equal to (!=)\n * ``GT``: Greater than (>)\n * ``LT``: Less than (<)\n * ``GTE``: Greater than or equal to (>=)\n * ``LTE``: Less than or equal to (<=)\n * ``IN``: Value in array\n * ``NOT_IN``: Value not in array\n * ``LIKE``: Pattern matching\n * ``ILIKE``: Case-insensitive pattern matching\n * ``ANY``: Any array element matches\n * ``ALL``: All array elements match\n * ``CONTAINS``: String contains value\n * ``EXISTS``: Field exists\n\n.. py:class:: FilterCondition\n\n Enumeration of conditions for combining multiple filters.\n\n Values:\n\n * ``AND``: All conditions must be true\n * ``OR``: Any condition must be true\n\n.. py:class:: MetadataFilter\n\n Represents a single metadata filter condition.\n\n Args:\n\n * ``key`` (str): The metadata field key to filter on\n * ``value`` (int | float | str): The value to compare against\n * ``operator`` (:py:class:`FilterOperator`): The comparison operator. Defaults to FilterOperator.EQ.\n\n.. py:class:: CompositeFilter\n\n Allows grouping multiple MetadataFilter instances using AND/OR conditions.\n\n Args:\n\n * ``filters`` (list[:py:class:`CompositeFilter` | :py:class:`MetadataFilter`]): List of filters to combine\n * ``condition`` (:py:class:`FilterCondition`): How to combine the filters. Defaults to FilterCondition.AND.\n\n.. py:function:: get_filter_clause(filters: CompositeFilter) -> str\n\n Get the filter clause for a given CompositeFilter.\n\n Args:\n\n * ``filters`` (:py:class:`CompositeFilter`): The composite filter to convert to a clause\n\n Returns:\n\n * str: The filter clause string for use in queries\n\n Raises:\n\n * ValueError: If an unknown operator or condition is encountered\n\n" }, { "path": "docs/reference/auth/built_in_ui.rst", "content": ".. _ref_guide_auth_built_in_ui:\n\n===========\nBuilt-in UI\n===========\n\n:edb-alt-title: Integrating Gel Auth's built-in UI\n\nTo use the built-in UI for Gel Auth, enable the built-in Auth UI by clicking\nthe \"Enable UI\" button under \"Login UI\" in the configuration section of the\n|Gel| UI. Set these configuration values:\n\n- ``redirect_to``: Once the authentication flow is complete, Gel will\n redirect the user's browser back to this URL in your application's\n backend.\n- ``redirect_to_on_signup``: If this is a new user, Gel will redirect\n the user's browser back to this URL in your application's backend.\n- ``app_name``: Used in the built-in UI to show the user the\n application's name in a few important places.\n- ``logo_url``: If provided, will show in the built-in UI as part of the\n page design.\n- ``dark_logo_url``: If provided and the user's system has indicated\n that they prefer a dark UI, this will show instead of ``logo_url`` in\n the built-in UI as part of the page design.\n- ``brand_color``: If provided, used in the built-in UI as part of the\n page design.\n\n\nExample Implementation\n======================\n\nWe will demonstrate the various steps below by building a NodeJS HTTP server in\na single file that we will use to simulate a typical web application.\n\n.. note::\n\n We are in the process of publishing helper libraries that you can use with\n popular languages and web frameworks. The details below show the inner\n workings of how data is exchanged with the Auth extension from a web app\n using HTTP. You can use this as a guide to integrate with your application\n written in any language that can send and receive HTTP requests.\n\nWe secure authentication tokens and other sensitive data by using PKCE\n(Proof Key of Code Exchange).\n\nStart the PKCE flow\n-------------------\n\nYour application server creates a 32-byte Base64 URL-encoded string (which will\nbe 43 bytes after encoding), called the ``verifier``. You need to store this\nvalue for the duration of the flow. One way to accomplish this bit of state is\nto use an HttpOnly cookie when the browser makes a request to the server for\nthis value, which you can then use to retrieve it from the cookie store at the\nend of the flow. Take this ``verifier`` string, hash it with SHA256, and then\nbase64url encode the resulting string. This new string is called the\n``challenge``.\n\n.. note::\n\n Since ``=`` is not a URL-safe character, if your Base64-URL encoding\n function adds padding, you should remove the padding before hashing the\n ``verifier`` to derive the ``challenge`` or when providing the ``verifier``\n or ``challenge`` in your requests.\n\n.. note::\n\n If you are familiar with PKCE, you will notice some differences from how RFC\n 7636 defines PKCE. Our authentication flow is not an OAuth flow, but rather a\n strict server-to-server flow with Proof Key of Code Exchange added for\n additional security to avoid leaking the authentication token. Here are some\n differences between PKCE as defined in RFC 7636 and our implementation:\n\n - We do not support the ``plain`` value for ``code_challenge_method``, and\n therefore do not read that value if provided in requests.\n - Our parameters omit the ``code_`` prefix, however we do support\n ``code_challenge`` and ``code_verifier`` as aliases, preferring\n ``challenge`` and ``verifier`` if present.\n\n.. code-block:: javascript\n\n import http from \"node:http\";\n import { URL } from \"node:url\";\n import crypto from \"node:crypto\";\n\n /**\n * You can get this value by running `gel instance credentials`.\n * Value should be:\n * `${protocol}://${host}:${port}/branch/${branch}/ext/auth/\n */\n const GEL_AUTH_BASE_URL = process.env.GEL_AUTH_BASE_URL;\n const SERVER_PORT = 3000;\n\n /**\n * Generate a random Base64 url-encoded string, and derive a \"challenge\"\n * string from that string to use as proof that the request for a token\n * later is made from the same user agent that made the original request\n *\n * @returns {Object} The verifier and challenge strings\n */\n const generatePKCE = () => {\n const verifier = crypto.randomBytes(32).toString(\"base64url\");\n\n const challenge = crypto\n .createHash(\"sha256\")\n .update(verifier)\n .digest(\"base64url\");\n\n return { verifier, challenge };\n };\n\n\n.. note::\n\n For |EdgeDB| versions before 5.0, the value for :gelenv:`AUTH_BASE_URL`\n in the above snippet should have the form:\n\n ``${protocol}://${host}:${port}/db/${database}/ext/auth/``\n\n\nLink to built-in UI\n-------------------\n\nNext, provide a link to your web application to either the ``/auth/ui/signin``\nor ``auth/ui/signup``. Those routes will generate the ``verifier`` and\n``challenge`` strings, save the ``verifier`` in a cookie and redirect the user\nto the built-in UI with the ``challenge`` in the search parameters.\n\n.. lint-off\n\n.. code-block:: javascript\n\n /**\n * In Node, the `req.url` is only the `pathname` portion of a URL. In\n * order to generate a full URL, we need to build the protocol and host\n * from other parts of the request.\n *\n * One reason we like to use `URL` objects here is to easily parse the\n * `URLSearchParams` from the request, and rather than do more error\n * prone string manipulation, we build a `URL`.\n *\n * @param {Request} req\n * @returns {URL}\n */\n const getRequestUrl = (req) => {\n const protocol = req.connection.encrypted ? \"https\" : \"http\";\n return new URL(req.url, `${protocol}://${req.headers.host}`);\n };\n\n const server = http.createServer(async (req, res) => {\n const requestUrl = getRequestUrl(req);\n\n switch (requestUrl.pathname) {\n case \"/auth/ui/signin\": {\n await handleUiSignIn(req, res);\n break;\n }\n\n case \"/auth/ui/signup\": {\n await handleUiSignUp(req, res);\n break;\n }\n\n case \"/auth/callback\": {\n await handleCallback(req, res);\n break;\n }\n\n default: {\n res.writeHead(404);\n res.end(\"Not found\");\n break;\n }\n }\n });\n\n /**\n * Redirects browser requests to Gel Auth UI sign in page with the\n * PKCE challenge, and saves PKCE verifier in an HttpOnly cookie.\n *\n * @param {Request} req\n * @param {Response} res\n */\n const handleUiSignIn = async (req, res) => {\n const { verifier, challenge } = generatePKCE();\n\n const redirectUrl = new URL(\"ui/signin\", GEL_AUTH_BASE_URL);\n redirectUrl.searchParams.set(\"challenge\", challenge);\n\n res.writeHead(301, {\n \"Set-Cookie\": `gel-pkce-verifier=${verifier}; HttpOnly; Path=/; Secure; SameSite=Strict`,\n Location: redirectUrl.href,\n });\n res.end();\n };\n\n /**\n * Redirects browser requests to Gel Auth UI sign up page with the\n * PKCE challenge, and saves PKCE verifier in an HttpOnly cookie.\n *\n * @param {Request} req\n * @param {Response} res\n */\n const handleUiSignUp = async (req, res) => {\n const { verifier, challenge } = generatePKCE();\n\n const redirectUrl = new URL(\"ui/signup\", GEL_AUTH_BASE_URL);\n redirectUrl.searchParams.set(\"challenge\", challenge);\n\n res.writeHead(301, {\n \"Set-Cookie\": `gel-pkce-verifier=${verifier}; HttpOnly; Path=/; Secure; SameSite=Strict`,\n Location: redirectUrl.href,\n });\n res.end();\n };\n\n server.listen(SERVER_PORT, () => {\n console.log(`HTTP server listening on port ${SERVER_PORT}...`);\n });\n\n\n.. lint-on\n\n\nRetrieve ``auth_token``\n-----------------------\n\nAt the very end of the flow, the Gel server will redirect the user's browser\nto the ``redirect_to`` address with a single query parameter: ``code``. This\nroute should be a server route that has access to the ``verifier``. You then\ntake that ``code`` and look up the ``verifier`` in the ``gel-pkce-verifier``\ncookie (``gel-pkce-verifier`` with |EdgeDB| <= 5), and make a request\nto the Gel Auth extension to exchange these two pieces of data for an\n``auth_token``.\n\n.. lint-off\n\n.. code-block:: javascript\n\n /**\n * Handles the PKCE callback and exchanges the `code` and `verifier\n * for an auth_token, setting the auth_token as an HttpOnly cookie.\n *\n * @param {Request} req\n * @param {Response} res\n */\n const handleCallback = async (req, res) => {\n const requestUrl = getRequestUrl(req);\n\n const code = requestUrl.searchParams.get(\"code\");\n if (!code) {\n const error = requestUrl.searchParams.get(\"error\");\n res.status = 400;\n res.end(\n `OAuth callback is missing 'code'. \\\n OAuth provider responded with error: ${error}`,\n );\n return;\n }\n\n const cookies = req.headers.cookie?.split(\"; \");\n const verifier = cookies\n ?.find((cookie) => cookie.startsWith(\"gel-pkce-verifier=\"))\n ?.split(\"=\")[1];\n if (!verifier) {\n res.status = 400;\n res.end(\n `Could not find 'verifier' in the cookie store. Is this the \\\n same user agent/browser that started the authorization flow?`,\n );\n return;\n }\n\n const codeExchangeUrl = new URL(\"token\", GEL_AUTH_BASE_URL);\n codeExchangeUrl.searchParams.set(\"code\", code);\n codeExchangeUrl.searchParams.set(\"verifier\", verifier);\n const codeExchangeResponse = await fetch(codeExchangeUrl.href, {\n method: \"GET\",\n });\n\n if (!codeExchangeResponse.ok) {\n const text = await codeExchangeResponse.text();\n res.status = 400;\n res.end(`Error from the auth server: ${text}`);\n return;\n }\n\n const { auth_token } = await codeExchangeResponse.json();\n res.writeHead(204, {\n \"Set-Cookie\": `gel-auth-token=${auth_token}; HttpOnly; Path=/; Secure; SameSite=Strict`,\n });\n res.end();\n };\n\n\n.. lint-on\n\n:ref:`Back to the Gel Auth guide `\n" }, { "path": "docs/reference/auth/email_password.rst", "content": ".. _ref_guide_auth_email_password:\n\n==================\nEmail and password\n==================\n\n:edb-alt-title: Integrating Gel Auth's email and password provider\n\nAlong with using the :ref:`built-in UI `, you can also\ncreate your own UI that calls to your own web application backend.\n\nUI considerations\n=================\n\nSimilar to how the built-in UI works, you can query the database configuration\nto discover which providers are configured and dynamically build the UI.\n\n.. code-block:: edgeql\n\n select cfg::Config.extensions[is ext::auth::AuthConfig].providers {\n name,\n [is ext::auth::OAuthProviderConfig].display_name,\n };\n\nThe ``name`` is a unique string that identifies the Identity Provider. OAuth\nproviders also have a ``display_name`` that you can use as a label for links or\nbuttons. In later steps, you'll be providing this ``name`` as the ``provider``\nin various endpoints.\n\n\nExample implementation\n======================\n\nWe will demonstrate the various steps below by building a NodeJS HTTP server in\na single file that we will use to simulate a typical web application. For this example,\nwe will require email verification to demonstrate the full flow, but you can\nconfigure your provider to not require verification by setting the\n``require_verification`` setting to ``false``.\n\n.. note::\n\n The details below show the inner workings of how data is exchanged with the\n Auth extension from a web app using HTTP. You can use this as a guide to\n integrate with your application written in any language that can send and\n receive HTTP requests.\n\n\nStart the PKCE flow\n-------------------\n\nWe secure authentication tokens and other sensitive data by using PKCE\n(Proof Key of Code Exchange).\n\nYour application server creates a 32-byte Base64 URL-encoded string (which will\nbe 43 bytes after encoding), called the ``verifier``. You need to store this\nvalue for the duration of the flow. One way to accomplish this bit of state is\nto use an HttpOnly cookie when the browser makes a request to the server for\nthis value, which you can then use to retrieve it from the cookie store at the\nend of the flow. Take this ``verifier`` string, hash it with SHA256, and then\nbase64url encode the resulting string. This new string is called the\n``challenge``.\n\n.. note::\n\n Since ``=`` is not a URL-safe character, if your Base64-URL encoding\n function adds padding, you should remove the padding before hashing the\n ``verifier`` to derive the ``challenge`` or when providing the ``verifier``\n or ``challenge`` in your requests.\n\n.. note::\n\n If you are familiar with PKCE, you will notice some differences from how RFC\n 7636 defines PKCE. Our authentication flow is not an OAuth flow, but rather a\n strict server-to-server flow with Proof Key of Code Exchange added for\n additional security to avoid leaking the authentication token. Here are some\n differences between PKCE as defined in RFC 7636 and our implementation:\n\n - We do not support the ``plain`` value for ``code_challenge_method``, and\n therefore do not read that value if provided in requests.\n - Our parameters omit the ``code_`` prefix, however we do support\n ``code_challenge`` and ``code_verifier`` as aliases, preferring\n ``challenge`` and ``verifier`` if present.\n\n.. lint-off\n\n.. code-block:: javascript\n\n import http from \"node:http\";\n import { URL } from \"node:url\";\n import crypto from \"node:crypto\";\n\n /**\n * You can get this value by running `gel instance credentials`.\n * Value should be:\n * `${protocol}://${host}:${port}/branch/${branch}/ext/auth/\n */\n const GEL_AUTH_BASE_URL = process.env.GEL_AUTH_BASE_URL;\n const SERVER_PORT = 3000;\n\n /**\n * Generate a random Base64 url-encoded string, and derive a \"challenge\"\n * string from that string to use as proof that the request for a token\n * later is made from the same user agent that made the original request\n *\n * @returns {Object} The verifier and challenge strings\n */\n const generatePKCE = () => {\n const verifier = crypto.randomBytes(32).toString(\"base64url\");\n\n const challenge = crypto\n .createHash(\"sha256\")\n .update(verifier)\n .digest(\"base64url\");\n\n return { verifier, challenge };\n };\n\n.. lint-on\n\n.. note::\n\n For |EdgeDB| versions before 5.0, the value for :gelenv:`AUTH_BASE_URL`\n in the above snippet should have the form:\n\n ``${protocol}://${host}:${port}/db/${database}/ext/auth/``\n\n\nSign-in and sign-up\n-------------------\n\nNext, we implement routes that handle registering a new user and authenticating\nan existing user.\n\n.. lint-off\n\n.. code-block:: javascript\n\n const server = http.createServer(async (req, res) => {\n const requestUrl = getRequestUrl(req);\n\n switch (requestUrl.pathname) {\n case \"/auth/signup\": {\n await handleSignUp(req, res);\n break;\n }\n\n case \"/auth/signin\": {\n await handleSignIn(req, res);\n break;\n }\n\n case \"/auth/verify\": {\n await handleVerify(req, res);\n break;\n }\n\n case \"/auth/send-password-reset-email\": {\n await handleSendPasswordResetEmail(req, res);\n break;\n }\n\n case \"/auth/ui/reset-password\": {\n await handleUiResetPassword(req, res);\n break;\n }\n\n case \"/auth/reset-password\": {\n await handleResetPassword(req, res);\n break;\n }\n\n default: {\n res.writeHead(404);\n res.end(\"Not found\");\n break;\n }\n }\n });\n\n /**\n * Handles sign up with email and password.\n *\n * @param {Request} req\n * @param {Response} res\n */\n const handleSignUp = async (req, res) => {\n let body = \"\";\n req.on(\"data\", (chunk) => {\n body += chunk.toString();\n });\n req.on(\"end\", async () => {\n const pkce = generatePKCE();\n const { email, password, provider } = JSON.parse(body);\n if (!email || !password || !provider) {\n res.status = 400;\n res.end(\n `Request body malformed. Expected JSON body with 'email', 'password', and 'provider' keys, but got: ${body}`,\n );\n return;\n }\n\n const registerUrl = new URL(\"register\", GEL_AUTH_BASE_URL);\n const registerResponse = await fetch(registerUrl.href, {\n method: \"post\",\n headers: {\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({\n challenge: pkce.challenge,\n email,\n password,\n provider,\n verify_url: `http://localhost:${SERVER_PORT}/auth/verify`,\n }),\n });\n\n if (!registerResponse.ok) {\n const text = await registerResponse.text();\n res.status = 400;\n res.end(`Error from the auth server: ${text}`);\n return;\n }\n\n const registerJson = await registerResponse.json();\n\n if (\"code\" in registerJson) {\n // No verification required, we can immediately get an auth token\n const tokenUrl = new URL(\"token\", GEL_AUTH_BASE_URL);\n tokenUrl.searchParams.set(\"code\", registerJson.code);\n tokenUrl.searchParams.set(\"verifier\", pkce.verifier);\n const tokenResponse = await fetch(tokenUrl.href, {\n method: \"get\",\n });\n\n if (!tokenResponse.ok) {\n const text = await tokenResponse.text();\n res.status = 400;\n res.end(`Error from the auth server: ${text}`);\n return;\n }\n\n const { auth_token } = await tokenResponse.json();\n res.writeHead(204, {\n \"Set-Cookie\": `gel-auth-token=${auth_token}; HttpOnly; Path=/; Secure; SameSite=Strict`,\n });\n\n res.end();\n } else {\n // Verification required, we need to render a notice to the user\n // to check their email for a verification link\n res.writeHead(200, { \"Content-Type\": \"text/html\" });\n res.end(`\n \n \n

Please check your email for a verification link.

\n \n \n `);\n }\n });\n };\n\n /**\n * Handles sign in with email and password.\n *\n * @param {Request} req\n * @param {Response} res\n */\n const handleSignIn = async (req, res) => {\n let body = \"\";\n req.on(\"data\", (chunk) => {\n body += chunk.toString();\n });\n req.on(\"end\", async () => {\n const pkce = generatePKCE();\n const { email, password, provider } = JSON.parse(body);\n if (!email || !password || !provider) {\n res.status = 400;\n res.end(\n `Request body malformed. Expected JSON body with 'email', 'password', and 'provider' keys, but got: ${body}`,\n );\n return;\n }\n\n const authenticateUrl = new URL(\"authenticate\", GEL_AUTH_BASE_URL);\n const authenticateResponse = await fetch(authenticateUrl.href, {\n method: \"post\",\n headers: {\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({\n challenge: pkce.challenge,\n email,\n password,\n provider,\n }),\n });\n\n if (!authenticateResponse.ok) {\n const text = await authenticateResponse.text();\n res.status = 400;\n res.end(`Error from the auth server: ${text}`);\n return;\n }\n\n const authenticateJson = await authenticateResponse.json();\n\n if (\"code\" in authenticateJson) {\n // User is verified, we can get an auth token\n const tokenUrl = new URL(\"token\", GEL_AUTH_BASE_URL);\n tokenUrl.searchParams.set(\"code\", authenticateJson.code);\n tokenUrl.searchParams.set(\"verifier\", pkce.verifier);\n const tokenResponse = await fetch(tokenUrl.href, {\n method: \"get\",\n });\n\n if (!tokenResponse.ok) {\n const text = await tokenResponse.text();\n res.status = 400;\n res.end(`Error from the auth server: ${text}`);\n return;\n }\n\n const { auth_token } = await tokenResponse.json();\n res.writeHead(204, {\n \"Set-Cookie\": `gel-auth-token=${auth_token}; HttpOnly; Path=/; Secure; SameSite=Strict`,\n });\n res.end();\n } else {\n // Verification required, we need to render a notice to the user\n // to check their email for a verification link\n res.writeHead(200, { \"Content-Type\": \"text/html\" });\n res.end(`\n \n \n

Please check your email for a verification link.

\n \n \n `);\n }\n });\n };\n\n.. lint-on\n\n\nEmail verification\n------------------\n\nWhen a new user signs up, by default we require them to verify their email\naddress before allowing the application to get an authentication token. To\nhandle the verification flow, we implement an endpoint:\n\n.. note::\n\n If your Email/Password provider uses the **Code** verification method, the\n verification email contains a one-time code rather than a link. In that\n case, prompt the user for the code and call ``POST /verify`` with:\n\n - **provider**: ``builtin::local_emailpassword``\n - **email** and **code**\n - optionally a **challenge** and **redirect_to** to receive a PKCE code or a redirect upon success\n\n The Link-based example below continues to work when the provider uses the\n Link method.\n\n.. note::\n\n šŸ’” If you would like to allow users to still log in, but offer limited access\n to your application, you can check the associated\n ``ext::auth::EmailPasswordFactor`` for the ``ext::auth::Identity`` to see if\n the ``verified_at`` property is some time in the past. You'll need to set\n the ``require_verification`` setting in the provider configuration to\n ``false``.\n\n.. lint-off\n\n.. code-block:: javascript\n\n /**\n * Handles the link in the email verification flow.\n *\n * @param {Request} req\n * @param {Response} res\n */\n const handleVerify = async (req, res) => {\n const requestUrl = getRequestUrl(req);\n const verification_token = requestUrl.searchParams.get(\"verification_token\");\n if (!verification_token) {\n res.status = 400;\n res.end(\n `Verify request is missing 'verification_token' search param. The verification email is malformed.`,\n );\n return;\n }\n\n const verifyUrl = new URL(\"verify\", GEL_AUTH_BASE_URL);\n const verifyResponse = await fetch(verifyUrl.href, {\n method: \"post\",\n headers: {\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({\n verification_token,\n provider: \"builtin::local_emailpassword\",\n }),\n });\n\n if (!verifyResponse.ok) {\n const text = await verifyResponse.text();\n res.status = 400;\n res.end(`Error from the auth server: ${text}`);\n return;\n }\n\n const { code } = await verifyResponse.json();\n\n const cookies = req.headers.cookie?.split(\"; \");\n const verifier = cookies\n ?.find((cookie) => cookie.startsWith(\"gel-pkce-verifier=\"))\n ?.split(\"=\")[1];\n if (verifier) {\n // Email verification flow is continuing from the original\n // user agent/browser, so we can immediately get an auth token\n const tokenUrl = new URL(\"token\", GEL_AUTH_BASE_URL);\n tokenUrl.searchParams.set(\"code\", code);\n tokenUrl.searchParams.set(\"verifier\", verifier);\n const tokenResponse = await fetch(tokenUrl.href, {\n method: \"get\",\n });\n\n if (!tokenResponse.ok) {\n const text = await tokenResponse.text();\n res.status = 400;\n res.end(`Error from the auth server: ${text}`);\n return;\n }\n\n const { auth_token } = await tokenResponse.json();\n res.writeHead(204, {\n \"Set-Cookie\": `gel-auth-token=${auth_token}; HttpOnly; Path=/; Secure; SameSite=Strict`,\n });\n res.end();\n return;\n }\n\n // Email verification flow is continuing from a different user agent/browser,\n // so we need to render a notice to the user to sign in, which will either\n // complete the PKCE flow or start a new one\n res.status = 200;\n res.end(\n `\n \n \n

Email verified! Please sign in to continue.

\n \n `,\n );\n };\n\n.. lint-on\n\n\nCreate a User object\n--------------------\n\nFor some applications, you may want to create a custom ``User`` type in the\ndefault module to attach application-specific information. You can tie this to\nan ``ext::auth::Identity`` by using the ``identity_id`` returned during the\nsign-up flow.\n\n.. note::\n\n For this example, we'll assume you have a one-to-one relationship between\n ``User`` objects and ``ext::auth::Identity`` objects. In your own\n application, you may instead decide to have a one-to-many relationship.\n\nGiven this ``User`` type:\n\n.. code-block:: sdl\n\n type User {\n email: str;\n name: str;\n\n required identity: ext::auth::Identity {\n constraint exclusive;\n };\n }\n\nYou can update the ``handleRegister`` function like this to create a new ``User``\nobject:\n\n.. lint-off\n\n.. code-block:: javascript-diff\n\n const registerJson = await registerResponse.json();\n\n + if (\"identity_id\" in registerJson) {\n + await client.query(`\n + with\n + identity := $identity_id,\n + emailFactor := (\n + select ext::auth::EmailFactor filter .identity = identity\n + ),\n + insert User {\n + email := emailFactor.email,\n + identity := identity\n + };\n + `, { identity_id: registerJson.identity_id });\n + }\n +\n if (\"code\" in registerJson) {\n\n.. lint-on\n\n\nPassword reset\n--------------\n\nTo allow users to reset their password, we implement three endpoints. The first\none sends the reset email. The second is the HTML form that is rendered when\nthe user follows the link in their email. And, the final one is the endpoint\nthat updates the password and logs in the user.\n\n.. note::\n\n If your provider is configured for the **Code** method for password reset,\n the email will contain a one-time code instead of a reset link/token. In\n that case:\n\n - Call ``POST /reset-password`` with **email**, **code**, **password** and\n optionally **challenge**.\n - If you include a **challenge**, the response will include a PKCE ``code``\n that you can exchange at ``POST /token`` to log the user in immediately.\n - If you omit **challenge**, the response will indicate success without a\n PKCE code and you should ask the user to sign in.\n\n.. lint-off\n\n.. code-block:: javascript\n\n /**\n * Request a password reset for an email.\n *\n * @param {Request} req\n * @param {Response} res\n */\n const handleSendPasswordResetEmail = async (req, res) => {\n let body = \"\";\n req.on(\"data\", (chunk) => {\n body += chunk.toString();\n });\n req.on(\"end\", async () => {\n const { email } = JSON.parse(body);\n const reset_url = `http://localhost:${SERVER_PORT}/auth/ui/reset-password`;\n const provider = \"builtin::local_emailpassword\";\n const pkce = generatePKCE();\n\n const sendResetUrl = new URL(\"send-reset-email\", GEL_AUTH_BASE_URL);\n const sendResetResponse = await fetch(sendResetUrl.href, {\n method: \"post\",\n headers: {\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({\n email,\n provider,\n reset_url,\n challenge: pkce.challenge,\n }),\n });\n\n if (!sendResetResponse.ok) {\n const text = await sendResetResponse.text();\n res.status = 400;\n res.end(`Error from auth server: ${text}`);\n return;\n }\n\n const { email_sent } = await sendResetResponse.json();\n\n res.writeHead(200, {\n \"Set-Cookie\": `gel-pkce-verifier=${pkce.verifier}; HttpOnly; Path=/; Secure; SameSite=Strict`,\n });\n res.end(`Reset email sent to '${email_sent}'`);\n });\n };\n\n /**\n * Render a simple reset password UI\n *\n * @param {Request} req\n * @param {Response} res\n */\n const handleUiResetPassword = async (req, res) => {\n const url = new URL(req.url);\n const reset_token = url.searchParams.get(\"reset_token\");\n res.writeHead(200, { \"Content-Type\": \"text/html\" });\n res.end(`\n \n \n
\n \n \n \n
\n \n \n `);\n };\n\n /**\n * Send new password with reset token to Gel Auth.\n *\n * @param {Request} req\n * @param {Response} res\n */\n const handleResetPassword = async (req, res) => {\n let body = \"\";\n req.on(\"data\", (chunk) => {\n body += chunk.toString();\n });\n req.on(\"end\", async () => {\n const { reset_token, password } = JSON.parse(body);\n if (!reset_token || !password) {\n res.status = 400;\n res.end(\n `Request body malformed. Expected JSON body with 'reset_token' and 'password' keys, but got: ${body}`\n );\n return;\n }\n const provider = \"builtin::local_emailpassword\";\n const cookies = req.headers.cookie.split(\"; \");\n const verifier = cookies\n .find((cookie) => cookie.startsWith(\"gel-pkce-verifier=\"))\n .split(\"=\")[1];\n if (!verifier) {\n res.status = 400;\n res.end(\n `Could not find 'verifier' in the cookie store. Is this the same user agent/browser that started the authorization flow?`\n );\n return;\n }\n const resetUrl = new URL(\"reset-password\", GEL_AUTH_BASE_URL);\n const resetResponse = await fetch(resetUrl.href, {\n method: \"post\",\n headers: {\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({\n reset_token,\n provider,\n password,\n }),\n });\n if (!resetResponse.ok) {\n const text = await resetResponse.text();\n res.status = 400;\n res.end(`Error from the auth server: ${text}`);\n return;\n }\n const { code } = await resetResponse.json();\n const tokenUrl = new URL(\"token\", GEL_AUTH_BASE_URL);\n tokenUrl.searchParams.set(\"code\", code);\n tokenUrl.searchParams.set(\"verifier\", verifier);\n const tokenResponse = await fetch(tokenUrl.href, {\n method: \"get\",\n });\n if (!tokenResponse.ok) {\n const text = await tokenResponse.text();\n res.status = 400;\n res.end(`Error from the auth server: ${text}`);\n return;\n }\n const { auth_token } = await tokenResponse.json();\n res.writeHead(204, {\n \"Set-Cookie\": `gel-auth-token=${auth_token}; HttpOnly; Path=/; Secure; SameSite=Strict`,\n });\n res.end();\n });\n };\n\n.. lint-on\n\n:ref:`Back to the Gel Auth guide `\n" }, { "path": "docs/reference/auth/http.rst", "content": ".. _ref_auth_http:\n\n========\nHTTP API\n========\n\nYour application server will interact with the Gel extension primarily by sending HTTP requests to the Gel server. This page describes the HTTP API exposed by the Gel server. For more in-depth guidance about integrating Gel Auth into your application, see :ref:`ref_guide_auth` for a reference example.\n\nThe following sections are organized by authentication type.\n\nResponses\n=========\n\nResponses typically include a JSON object that include a ``code`` property that can be exchanged for an access token by providing the matching PKCE verifier associated with the ``code``. Some endpoints can be configured to return responses as redirects and include response data in the redirect location's query string.\n\nGeneral\n=======\n\nPOST /token\n-----------\n\nExchanges a PKCE authorization code (obtained from a successful registration, authentication, or email verification flow that included a PKCE challenge) for a session token.\n\n**Request Parameters (Query String):**\n\n* ``code`` (string, required): The PKCE authorization code that was previously issued.\n* ``verifier`` (string, required, also accepts ``code_verifier``): The PKCE code verifier string (plaintext, typically 43-128 characters) that was originally used to generate the ``code_challenge``.\n\n**Response:**\n\n1. **Successful Token Exchange:**\n\n * This occurs if the ``code`` is valid, and the provided ``verifier`` correctly matches the ``challenge`` associated with the ``code``.\n * The PKCE ``code`` is consumed and cannot be reused.\n * A 200 OK response is returned with a JSON body containing the session token and identity information:\n\n .. code-block:: json\n\n {\n \"auth_token\": \"your_new_session_jwt\",\n \"identity_id\": \"the_users_identity_id\",\n \"provider_token\": \"optional_oauth_provider_access_token\",\n \"provider_refresh_token\": \"optional_oauth_provider_refresh_token\",\n \"provider_id_token\": \"optional_oauth_provider_id_token\"\n }\n\n .. note::\n\n ``provider_token``, ``provider_refresh_token``, and ``provider_id_token`` are only populated if the PKCE flow originated from an interaction with an external OAuth provider that returned these tokens.\n\n2. **PKCE Verification Failed:**\n\n * The ``code`` was found, but the ``verifier`` did not match the stored challenge.\n * An HTTP error response 403 Forbidden with a JSON body indicating ``PKCEVerificationFailed``.\n\n3. **Unknown Code:**\n\n * The provided ``code`` was not found.\n * An HTTP error response 403 Forbidden with a JSON body indicating \"NoIdentityFound\".\n\n4. **Code found, but not associated with an Identity:**\n\n * The ``code`` was found, but it is not associated with a user identity.\n * An HTTP error response 400 Bad Request with a JSON body indicating \"InvalidData\".\n\n5. **Invalid Verifier Length:**\n\n * The ``verifier`` string is shorter than 43 characters or longer than 128 characters.\n * An HTTP 400 Bad Request response with a JSON body detailing the length requirement.\n\n6. **Missing Parameters:**\n\n * Either ``code`` or ``verifier`` (or ``code_verifier``) is missing from the query string.\n * An HTTP 400 Bad Request response with a JSON body indicating the missing parameter.\n\nEmail and password\n==================\n\nPOST /register\n--------------\n\nRegister a new user with email and password.\n\n**Request Body (JSON):**\n\n* ``email`` (string, required): The user's email address.\n* ``password`` (string, required): The user's desired password.\n* ``provider`` (string, required): The name of the provider to use: ``builtin::local_emailpassword``\n* ``challenge`` (string, optional): A PKCE code challenge. This is required if the provider is configured with ``require_verification: false`` since registering will also authenticate and authentication is protected by a PKCE code exchange.\n* ``redirect_to`` (string, optional): A URL to redirect to upon successful registration.\n* ``verify_url`` (string, optional): The base URL for the email verification link. If not provided, it defaults to ``/ui/verify``, the built-in UI endpoint for verifying email addresses. The verification token will be appended as a query parameter to this URL.\n* ``redirect_on_failure`` (string, optional): A URL to redirect to if registration fails.\n\n.. note::\n\n The verification email sent after registration depends on your provider's verification method:\n\n - **Code**: users receive a one-time code and must call ``POST /verify`` with ``provider``, ``email`` and the ``code``.\n - **Link**: users receive a verification link that carries a ``verification_token`` and must call ``POST /verify`` with ``provider`` and the ``verification_token`` (often done by following the link).\n\n**Response:**\n\nThe behavior of the response depends on the request parameters and server-side provider configuration (specifically, ``require_verification``).\n\n1. **Successful Registration with Email Verification Required:**\n\n * This occurs if the provider has ``require_verification: true``.\n * If ``redirect_to`` is provided in the request:\n\n * A 302 redirect to the ``redirect_to`` URL occurs.\n * The redirect URL will include ``identity_id`` and ``verification_email_sent_at`` as query parameters.\n\n * If ``redirect_to`` is NOT provided:\n\n * A 201 Created response is returned with a JSON body:\n\n .. code-block:: json\n\n {\n \"identity_id\": \"...\",\n \"verification_email_sent_at\": \"YYYY-MM-DDTHH:MM:SS.ffffffZ\"\n }\n\n2. **Successful Registration with Email Verification NOT Required (PKCE Flow):**\n\n * This occurs if the provider has ``require_verification: false``. The ``challenge`` parameter is mandatory in the request.\n * If ``redirect_to`` is provided in the request:\n\n * A 302 redirect to the ``redirect_to`` URL occurs.\n * The redirect URL will include ``code`` (the PKCE authorization code) and ``provider`` as query parameters.\n\n * If ``redirect_to`` is NOT provided:\n\n * A 201 Created response is returned with a JSON body:\n\n .. code-block:: json\n\n {\n \"code\": \"...\",\n \"provider\": \"...\"\n }\n\n3. **Registration Failure:**\n\n * If ``redirect_on_failure`` is provided in the request and is an allowed URL:\n\n * A 302 redirect to the ``redirect_on_failure`` URL occurs.\n * The redirect URL will include ``error`` (a description of the error) and ``email`` (the submitted email) as query parameters.\n\n * Otherwise (no ``redirect_on_failure`` or it's not allowed):\n\n * An HTTP error response (e.g., 400 Bad Request, 500 Internal Server Error) is returned with a JSON body describing the error. For example:\n\n .. code-block:: json\n\n {\n \"message\": \"Error description\",\n \"type\": \"ErrorType\",\n \"code\": \"ERROR_CODE\"\n }\n\n**Common Error Scenarios:**\n\n* Missing ``provider`` in the request.\n* Missing ``challenge`` in the request when the provider has ``require_verification: false``.\n* Email already exists.\n* Invalid password (e.g., too short, if policies are enforced).\n\nPOST /authenticate\n------------------\n\nAuthenticate a user using email and password.\n\n**Request Body (JSON):**\n\n* ``email`` (string, required): The user's email address.\n* ``password`` (string, required): The user's password.\n* ``provider`` (string, required): The name of the provider to use: ``builtin::local_emailpassword``\n* ``challenge`` (string, required): A PKCE code challenge.\n* ``redirect_to`` (string, optional): A URL to redirect to upon successful authentication.\n* ``redirect_on_failure`` (string, optional): A URL to redirect to if authentication fails. If not provided, but ``redirect_to`` is, ``redirect_to`` will be used as the fallback for failure redirection.\n\n**Response:**\n\nThe behavior of the response depends on the request parameters and the outcome of the authentication attempt.\n\n1. **Successful Authentication:**\n\n * A PKCE authorization code is generated and associated with the user's session.\n * If ``redirect_to`` is provided in the request:\n\n * A 302 redirect to the ``redirect_to`` URL occurs.\n * The redirect URL will include a ``code`` (the PKCE authorization code) as a query parameter.\n\n * If ``redirect_to`` is NOT provided:\n\n * A 200 OK response is returned with a JSON body:\n\n .. code-block:: json\n\n {\n \"code\": \"...\"\n }\n\n2. **Authentication Failure (e.g., invalid credentials, user not found):**\n\n * If ``redirect_on_failure`` (or ``redirect_to`` as a fallback) is provided in the request and is an allowed URL:\n\n * A 302 redirect to this URL occurs.\n * The redirect URL will include ``error`` (a description of the error) and ``email`` (the submitted email) as query parameters.\n\n * Otherwise (no applicable redirect URL or it's not allowed):\n\n * An HTTP error response (e.g., 400, 401) is returned with a JSON body describing the error. For example:\n\n .. code-block:: json\n\n {\n \"message\": \"Invalid credentials\",\n \"type\": \"InvalidCredentialsError\",\n \"code\": \"INVALID_CREDENTIALS\"\n }\n\n3. **Email Verification Required:**\n\n * This occurs if the provider is configured with ``require_verification: true`` and the user has not yet verified their email address.\n * The response follows the same logic as **Authentication Failure**:\n\n * If ``redirect_on_failure`` (or ``redirect_to``) is provided, a redirect occurs with an error like \"VerificationRequired\".\n * Otherwise, an HTTP error (often 403 Forbidden) is returned with a JSON body indicating that email verification is required.\n\n**Common Error Scenarios:**\n\n* Missing required fields in the request: ``email``, ``password``, ``provider``, or ``challenge``.\n* Invalid email or password.\n* User account does not exist.\n* User account exists but email is not verified (if ``require_verification: true`` for the provider).\n\nPOST /send-reset-email\n----------------------\n\nSend a password reset email to a user.\n\n**Request Body (JSON):**\n\n* ``provider`` (string, required): The name of the provider: ``builtin::local_emailpassword``.\n* ``email`` (string, required): The email address of the user requesting the password reset.\n* ``reset_url`` (string, required): The base URL for the password reset page (used for the Link method). The ``reset_token`` will be appended as a query parameter. This URL must be an allowed redirect URI in the server configuration.\n* ``challenge`` (string, required): A PKCE code challenge. For the Link method it is embedded in the ``reset_token``; for the Code method it can be re-used later when completing the reset to obtain a PKCE code.\n* ``redirect_to`` (string, optional): A URL to redirect to after the reset email has been successfully queued for sending.\n* ``redirect_on_failure`` (string, optional): A URL to redirect to if there's an error during the process. If not provided, but ``redirect_to`` is, ``redirect_to`` will be used as the fallback for failure redirection.\n\n.. note::\n\n The email sent depends on your provider's configuration:\n\n - **Link**: a reset link is sent containing a ``reset_token``; the user should then call ``POST /reset-password`` with this token.\n - **Code**: a one-time code is sent to the email address; the user should then call ``POST /reset-password`` with ``email`` and ``code`` (and optionally ``challenge`` to receive a PKCE code).\n\n**Response:**\n\nThe endpoint always attempts to respond in a way that does not reveal whether an email address is registered or not.\n\n1. **Reset Email Queued (or User Not Found):**\n\n * If the user exists, a password reset email is generated and sent.\n * If the user does not exist, the server simulates a successful send to prevent email enumeration attacks.\n * If ``redirect_to`` is provided in the request:\n\n * A 302 redirect to the ``redirect_to`` URL occurs.\n * The redirect URL will include ``email_sent`` (the email address provided in the request) as a query parameter.\n\n * If ``redirect_to`` is NOT provided:\n\n * A 200 OK response is returned with a JSON body:\n\n .. code-block:: json\n\n {\n \"email_sent\": \"user@example.com\"\n }\n\n2. **Failure (e.g., ``reset_url`` not allowed, SMTP server error):**\n\n * This occurs for errors not related to whether the user exists, such as configuration issues or mail server problems.\n * If ``redirect_on_failure`` (or ``redirect_to`` as a fallback) is provided in the request and is an allowed URL:\n\n * A 302 redirect to this URL occurs.\n * The redirect URL will include ``error`` (a description of the error) and ``email`` (the submitted email) as query parameters.\n\n * Otherwise (no applicable redirect URL or it's not allowed):\n\n * An HTTP error response (e.g., 400 Bad Request, 500 Internal Server Error) is returned with a JSON body describing the error.\n\n**Common Error Scenarios (leading to the Failure response):**\n\n* Missing required fields in the request: ``provider``, ``email``, ``reset_url``, or ``challenge``.\n* The provided ``reset_url`` is not in the server's list of allowed redirect URIs.\n* Internal server error during email dispatch (e.g., SMTP configuration issues).\n\nPOST /reset-password\n--------------------\n\nResets a user's password using a reset token and a new password. This endpoint completes the password reset flow initiated by ``POST /send-reset-email``.\n\n**Request Body (JSON):**\n\n* ``provider`` (string, required): The name of the provider: ``builtin::local_emailpassword``.\n* ``password`` (string, required): The new password for the user's account.\n\nChoose one of the following modes:\n\n- **Token mode (Link method)**\n\n * ``reset_token`` (string, required): The token that was emailed to the user.\n\n- **Code mode**\n\n * ``email`` (string, required): The user's email address.\n * ``code`` (string, required): The one-time code sent by email.\n * ``challenge`` (string, optional): If provided, a PKCE authorization code will be generated upon success.\n\nOptional for both modes:\n\n* ``redirect_to`` (string, optional): A URL to redirect to after the password has been successfully reset. If provided and a PKCE code is generated, it will be appended as a query parameter.\n* ``redirect_on_failure`` (string, optional): A URL to redirect to if the password reset process fails. If not provided, but ``redirect_to`` is, ``redirect_to`` will be used as the fallback.\n\n**Response:**\n\n- **Token mode (Link method)**\n\n * The ``reset_token`` is validated, and the user's password is updated.\n * A PKCE authorization ``code`` is generated using the challenge embedded in the token.\n * If ``redirect_to`` is provided, a 302 redirect occurs with ``code`` appended; otherwise, a 200 OK JSON response is returned with ``{\"code\": \"...\"}``.\n\n- **Code mode**\n\n * The ``email``/``code`` are validated, and the user's password is updated.\n * If a ``challenge`` is provided, a PKCE authorization ``code`` is generated.\n * If ``redirect_to`` is provided and a PKCE code was generated, a 302 redirect occurs with ``code`` appended; if ``challenge`` was not provided, a 200 OK JSON response is returned with ``{\"status\": \"password_reset\"}``.\n\n- **Failure (invalid inputs or server error)**\n\n * If ``redirect_on_failure`` (or ``redirect_to`` as a fallback) is provided and is an allowed URL, a 302 redirect occurs with an ``error`` parameter (and submitted ``reset_token``/``email`` where applicable).\n * Otherwise, an HTTP error response is returned with a JSON error body (e.g., 400, 403, 500).\n\n**Common Error Scenarios:**\n\n* Missing required fields in the request: ``provider``, ``reset_token``, or ``password``.\n* The ``reset_token`` is malformed, has an invalid signature, or is expired.\n* Internal server error during the password update process.\n\nEmail verification\n==================\n\nThese endpoints apply to the Email and password provider, as well as the WebAuthn provider. Verification emails are sent even if you do not *require* verification. The difference between requiring verification and not is that if you require verification, the user must verify their email before they can authenticate. If you do not require verification, the user can authenticate without verifying their email.\n\nPOST /verify\n------------\n\nVerify a user's email address. Supports both Link and Code methods.\n\n**Request Body (JSON):**\n\n* ``provider`` (string, required): The provider name, e.g., ``builtin::local_emailpassword`` or ``builtin::local_webauthn``.\n\nChoose exactly one verification mode:\n\n- **Link mode**\n\n * ``verification_token`` (string, required): The JWT sent to the user (typically via an email link) to verify their email.\n\n- **Code mode**\n\n * ``email`` (string, required): The user's email address to verify.\n * ``code`` (string, required): The one-time code sent via email.\n * ``challenge`` (string, optional, also accepts ``code_challenge``): If provided, a PKCE authorization code will be generated upon success.\n * ``redirect_to`` (string, optional): If provided, a redirect response will be sent upon success. This URL must be in the server's list of allowed redirect URIs.\n\n**Response:**\n\n- **Link mode**\n\n The primary action is to validate the ``verification_token`` and mark the associated email as verified. The exact response depends on the contents of the ``verification_token`` (it may include a PKCE challenge and/or a redirect URL specified during its creation):\n\n 1. With challenge and redirect URL in token\n\n * A PKCE authorization code is generated using the challenge from the token.\n * A 302 redirect to the URL specified in the token (``maybe_redirect_to``) occurs, with ``code`` appended as a query parameter.\n\n 2. With challenge only in token\n\n * A PKCE authorization code is generated using the challenge from the token.\n * A 200 OK response is returned with a JSON body:\n\n .. code-block:: json\n\n {\n \"code\": \"generated_pkce_code\"\n }\n\n 3. With redirect URL only in token\n\n * A 302 redirect to the URL specified in the token (``maybe_redirect_to``) occurs (no ``code`` is added).\n\n 4. No challenge or redirect URL in token\n\n * A 204 No Content response is returned.\n\n 5. Invalid or expired token\n\n * A 403 Forbidden response is returned with a JSON body (e.g., token expired).\n\n- **Code mode**\n\n After validating ``email`` and ``code`` and marking the email as verified, behavior depends on optional ``challenge`` and ``redirect_to``:\n\n 1. ``challenge`` and ``redirect_to`` provided\n\n * A PKCE authorization code is generated and a 302 redirect to ``redirect_to`` occurs with ``code`` appended as a query parameter.\n\n 2. Only ``challenge`` provided\n\n * A PKCE authorization code is generated and a 200 OK response is returned with a JSON body:\n\n .. code-block:: json\n\n {\n \"code\": \"generated_pkce_code\"\n }\n\n 3. Only ``redirect_to`` provided\n\n * A 302 redirect to ``redirect_to`` occurs (no PKCE code is generated).\n\n 4. Neither provided\n\n * A 204 No Content response is returned.\n\n**Common Error Scenarios:**\n\n* Missing ``provider`` or ``verification_token`` in the request (results in HTTP 400).\n* The ``verification_token`` is malformed, has an invalid signature, or is expired (results in HTTP 403).\n* An internal error occurs while trying to update the email verification status (results in HTTP 500).\n\nPOST /resend-verification-email\n-------------------------------\n\nResend a verification email to a user. This can be useful if the original email was lost or the token expired.\n\n**Request Body (JSON):**\n\nThe request must include ``provider`` and a way to identify the user's email factor.\n\n* ``provider`` (string, required): The provider name, e.g., ``builtin::local_emailpassword`` or ``builtin::local_webauthn``.\n\nThen, choose **one** of the following methods to specify the user:\n\n* **Method 1: Using an existing Verification Token**\n\n * ``verification_token`` (string): An old (even expired) verification token. The system will extract necessary details (like ``identity_id``, original ``verify_url``, ``challenge``, and ``redirect_to``) from this token to generate a new one.\n\n* **Method 2: Using Email Address (for Email/Password provider)**\n\n * ``email`` (string, required if ``provider`` is ``builtin::local_emailpassword`` and ``verification_token`` is not used): The user's email address.\n * ``verify_url`` (string, optional): The base URL for the new verification link. Defaults to the server's configured UI verify path (e.g., ``/ui/verify``).\n * ``challenge`` (string, optional, also accepts ``code_challenge``): A PKCE code challenge to be embedded in the new verification token.\n * ``redirect_to`` (string, optional): A URL to redirect to after successful verification using the new token. This URL must be in the server's list of allowed redirect URIs.\n\n* **Method 3: Using WebAuthn Credential ID (for WebAuthn provider)**\n\n * ``credential_id`` (string, required if ``provider`` is ``builtin::local_webauthn`` and ``verification_token`` is not used): The Base64 encoded WebAuthn credential ID.\n * ``verify_url`` (string, optional): As above.\n * ``challenge`` (string, optional, also accepts ``code_challenge``): As above.\n * ``redirect_to`` (string, optional): As above. This URL must be in the server's list of allowed redirect URIs.\n\n**Response:**\n\nThe endpoint aims to prevent email enumeration by always returning a successful status code if the request format is valid, regardless of whether the user or email factor was found.\n\n1. **Verification Email Queued (or User/Email Factor Not Found):**\n\n * If the user/email factor is found, a new verification email with a fresh token is generated and sent.\n * If the user/email factor is not found (based on the provided identifier), the server simulates a successful send.\n * A 200 OK response is returned. The response body is typically empty.\n\n2. **Failure (Invalid Request or Server Error):**\n\n * If the request is malformed (e.g., unsupported ``provider``, ``redirect_to`` URL not allowed, missing required fields for the chosen identification method), an HTTP 400 Bad Request with a JSON error body is returned.\n * If an internal server error occurs (e.g., SMTP issues), an HTTP 500 Internal Server Error with a JSON error body is returned.\n\n**Common Error Scenarios:**\n\n* Unsupported ``provider`` name.\n* Missing ``verification_token`` when it's the chosen method, or missing ``email`` / ``credential_id`` for other methods.\n* Providing a ``redirect_to`` URL that is not in the allowed list.\n* Internal SMTP errors preventing email dispatch.\n\n.. note::\n\n If the provider uses the **Code** verification method, the resend email will contain a one-time code instead of a link. In this case, ``verify_url``, ``challenge``, and ``redirect_to`` are not included in the email and are only relevant for the Link method.\n\nOAuth\n=====\n\nPOST /authorize\n---------------\n\nInitiate an OAuth authorization flow.\n\n**Request Parameters (Query String):**\n\n* ``provider`` (string, required): The name of the OAuth provider to use (e.g., ``builtin::oauth::google``).\n* ``redirect_to`` (string, required): The URL to redirect to after a successful OAuth flow completes and a PKCE code is obtained. This URL must be in the server's list of allowed redirect URIs.\n* ``challenge`` (string, required, also accepts ``code_challenge``): A PKCE code challenge generated by your application.\n* ``redirect_to_on_signup`` (string, optional): An alternative URL to redirect to after a *new* user successfully completes the OAuth flow. If not provided, ``redirect_to`` will be used for both new and existing users. This URL must also be in the server's list of allowed redirect URIs.\n* ``callback_url`` (string, optional): The URL the OAuth provider should redirect back to after the user authorizes the application. If not provided, it defaults to ``/callback``. This URL must be in the server's list of allowed redirect URIs.\n\n**Response:**\n\n1. **Successful Authorization Initiation:**\n\n * The server generates a PKCE challenge record and prepares for the OAuth flow.\n * A 302 Found redirect response is returned.\n * The ``Location`` header will contain the authorization URL provided by the external OAuth identity provider. The user's browser will be directed to this URL to begin the OAuth provider's authentication/authorization process.\n\n**Common Error Scenarios:**\n\n* Missing required fields in the query string: ``provider``, ``redirect_to``, or ``challenge``.\n* The provided ``redirect_to``, ``redirect_to_on_signup``, or ``callback_url`` is not in the server's list of allowed redirect URIs.\n* Configuration error on the server (e.g., the specified provider is not configured).\n\nPOST /callback\n--------------\n\nHandle the redirect from the OAuth provider. This endpoint is typically called by the OAuth provider after the user has completed the authentication and authorization process on the provider's site. It processes the response from the provider, exchanges the authorization code for Gel session information (and potentially provider tokens), and redirects the user back to the application.\n\nThis endpoint accepts parameters either in the query string (for GET requests) or in the request body as ``application/x-www-form-urlencoded`` (for POST requests).\n\n**Request Parameters (Query String or Form Data):**\n\n* ``state`` (string, required): The state parameter originally sent in the ``POST /authorize`` request. This is a signed JWT containing information needed to complete the flow (like provider name, redirect URLs, and the PKCE challenge).\n* ``code`` (string, optional): The authorization code provided by the OAuth identity provider. This is present on successful authorization.\n* ``error`` (string, optional): An error code provided by the OAuth identity provider, if authorization failed.\n* ``error_description`` (string, optional): A human-readable description of the error provided by the OAuth identity provider.\n\n**Response:**\n\n1. **Successful Callback and Token Exchange:**\n\n * This occurs when the OAuth provider returns a ``code``, and the ``state`` is valid.\n * The server exchanges the OAuth code for identity information and potentially provider access/refresh tokens.\n * The identity is linked to the PKCE challenge provided in the original ``state``.\n * A 302 Found redirect response is returned.\n * The ``Location`` header will contain the ``redirect_to`` (or ``redirect_to_on_signup`` if applicable) URL specified in the original ``state`` parameter.\n * The redirect URL will include the Gel PKCE authorization ``code`` and the ``provider`` name as query parameters (e.g., ``https://app.example.com/success?code=gel_pkce_code&provider=oauth_provider_name``). This PKCE code can then be exchanged for a session token via ``POST /token``.\n\n2. **OAuth Provider Returned an Error:**\n\n * This occurs when the OAuth provider redirects back with an ``error`` parameter.\n * A 302 Found redirect response is returned.\n * The ``Location`` header will contain the ``redirect_to`` URL specified in the original ``state`` parameter.\n * The redirect URL will include the ``error`` and optionally ``error_description`` and the user's ``email`` (if available and relevant) as query parameters.\n\n**Common Error Scenarios (before redirect):**\n\n* Missing ``state`` parameter in the request.\n* Invalid or malformed ``state`` token.\n* The OAuth provider did not return either a ``code`` or an ``error``.\n* Errors during the server's exchange of the OAuth code with the provider (these typically result in an HTTP error response from this endpoint rather than a redirect with an error).\n\nWebAuthn\n========\n\nPOST /webauthn/register\n-----------------------\n\nRegister a new WebAuthn credential for a user. This typically follows a call to ``GET /webauthn/register/options`` where the registration options were obtained.\n\n**Request Body (JSON):**\n\n* ``provider`` (string, required): The name of the WebAuthn provider to use: ``builtin::local_webauthn``.\n* ``challenge`` (string, required): A PKCE code challenge. This challenge will be linked to the identity upon successful registration if email verification is not required.\n* ``email`` (string, required): The user's email address associated with the WebAuthn credential.\n* ``credentials`` (string, required): The credential data obtained from the client-side WebAuthn API (``navigator.credentials.create()``). This should be a JSON string.\n* ``verify_url`` (string, required): The base URL for the email verification link that will be emailed to the user if email verification is required.\n* ``user_handle`` (string, optional): The Base64 URL encoded user handle generated during the options request. This can also be passed via a cookie named ``edgedb-webauthn-registration-user-handle``.\n\n**Request Cookies:**\n\n* ``edgedb-webauthn-registration-user-handle`` (string, optional): The Base64 URL encoded user handle generated during the options request. If present, this overrides the ``user_handle`` in the request body.\n\n**Response:**\n\nThe response depends on whether the WebAuthn provider is configured to require email verification or not.\n\n1. **Successful Registration with Email Verification Required:**\n\n * A 201 Created response is returned with a JSON body:\n\n .. code-block:: json\n\n {\n \"identity_id\": \"...\",\n \"verification_email_sent_at\": \"YYYY-MM-DDTHH:MM:SS.ffffffZ\"\n }\n\n * The ``edgedb-webauthn-registration-user-handle`` cookie is cleared.\n\n2. **Successful Registration with Email Verification NOT Required (PKCE Flow):**\n\n * A 201 Created response is returned with a JSON body:\n\n .. code-block:: json\n\n {\n \"code\": \"...\",\n \"provider\": \"builtin::local_webauthn\"\n }\n\n * The ``edgedb-webauthn-registration-user-handle`` cookie is cleared. The returned ``code`` can be exchanged for a session token at the ``POST /token`` endpoint.\n\n**Common Error Scenarios:**\n\n* Missing required fields in the request body or user handle (either in body or cookie).\n* Invalid or malformed ``credentials`` or ``user_handle`` data.\n* The specified ``verify_url`` is not in the server's list of allowed redirect URIs.\n* Errors during the WebAuthn registration process on the server (e.g., credential already registered).\n* Configuration error on the server (e.g., WebAuthn provider not configured).\n\nPOST /webauthn/authenticate\n---------------------------\n\nAuthenticate a user using an existing WebAuthn credential. This typically follows a call to ``GET /webauthn/authenticate/options`` where the authentication options were obtained.\n\n**Request Body (JSON):**\n\n* ``provider`` (string, required): The name of the WebAuthn provider to use: ``builtin::local_webauthn``.\n* ``challenge`` (string, required): A PKCE code challenge. This challenge will be linked to the authenticated identity upon successful authentication.\n* ``email`` (string, required): The user's email address associated with the WebAuthn credential they are attempting to use.\n* ``assertion`` (string, required): The assertion data obtained from the client-side WebAuthn API (``navigator.credentials.get()``). This should be a JSON string.\n\n**Response:**\n\n1. **Successful Authentication:**\n\n * This occurs when the provided ``assertion`` successfully verifies the user's identity based on the provided ``email``.\n * If email verification is required for the provider, the user's email must also be verified.\n * A PKCE authorization ``code`` is generated and linked to the authenticated identity using the provided ``challenge``.\n * A 200 OK response is returned with a JSON body:\n\n .. code-block:: json\n\n {\n \"code\": \"...\"\n }\n\n * The returned ``code`` can be exchanged for a session token at the ``POST /token`` endpoint.\n\n2. **Authentication Failure:**\n\n * This occurs if the provided ``assertion`` does not match the registered credential for the given email, the email is not found, or if email verification is required but the email is not verified.\n * An HTTP error response (e.g., 401 Unauthorized or 403 Forbidden) is returned with a JSON body describing the error (e.g., \"Failed to authenticate WebAuthn\", \"VerificationRequired\").\n\n**Common Error Scenarios:**\n\n* Missing required fields in the request body: ``challenge``, ``email``, or ``assertion``.\n* Invalid or malformed ``assertion`` data.\n* No WebAuthn credential found for the provided email.\n* WebAuthn authentication failed (e.g., invalid signature).\n* Email verification is required for the provider, but the user's email is not verified.\n* Configuration error on the server (e.g., WebAuthn provider not configured).\n\nGET /webauthn/register/options\n------------------------------\n\nGet the necessary options from the server to initiate a WebAuthn registration ceremony on the client side (using ``navigator.credentials.create()``).\n\n**Request Parameters (Query String):**\n\n* ``email`` (string, required): The user's email address for whom registration options are being requested.\n\n**Response:**\n\n1. **Successful Options Retrieval:**\n\n * A 200 OK response is returned.\n * The ``Content-Type`` header is ``application/json``.\n * The response body contains a JSON object with the WebAuthn registration options, compatible with the Web Authentication API (``PublicKeyCredentialCreationOptions``).\n * A cookie named ``edgedb-webauthn-registration-user-handle`` is set containing the Base64 URL encoded user handle generated by the server. This cookie is needed for the subsequent ``POST /webauthn/register`` request.\n\n**Common Error Scenarios:**\n\n* Missing required ``email`` query parameter.\n* Configuration error on the server (e.g., WebAuthn provider not configured).\n* Errors during the generation of registration options on the server.\n\nGET /webauthn/authenticate/options\n----------------------------------\n\nGet the necessary options from the server to initiate a WebAuthn authentication ceremony on the client side (using ``navigator.credentials.get()``).\n\n**Request Parameters (Query String):**\n\n* ``email`` (string, required): The user's email address for whom authentication options are being requested. The server will look up associated WebAuthn credentials based on this email.\n\n**Response:**\n\n1. **Successful Options Retrieval:**\n\n * A 200 OK response is returned.\n * The ``Content-Type`` header is ``application/json``.\n * The response body contains a JSON object with the WebAuthn authentication options, compatible with the Web Authentication API (``PublicKeyCredentialRequestOptions``). These options will include information about the user's registered credentials to challenge the client.\n\n**Common Error Scenarios:**\n\n* Missing required ``email`` query parameter.\n* Configuration error on the server (e.g., WebAuthn provider not configured).\n* Errors during the generation of authentication options on the server (e.g., no credentials found for the email).\n\nMagic link\n==========\n\nPOST /magic-link/register\n-------------------------\n\nRegisters a new user with a magic link credential and sends a magic link email to their email address.\n\n**Request Body (JSON or application/x-www-form-urlencoded):**\n\nThe required fields depend on the provider's verification method.\n\n- **Code method**\n\n * ``email`` (string, required): The user's email address.\n * ``redirect_to`` (string, optional): A URL to redirect to after the email has been queued. If omitted, the request must accept ``application/json``.\n\n- **Link method**\n\n * ``email`` (string, required): The user's email address.\n * ``challenge`` (string, required): A PKCE code challenge that will be embedded in the magic link token.\n * ``callback_url`` (string, required): The URL that the user will be redirected to after clicking the magic link in the email. A PKCE authorization ``code`` will be appended to this URL. This URL must be in the server's list of allowed redirect URIs.\n * ``redirect_on_failure`` (string, required): A URL to redirect to if there's an error during the registration or email sending process. Error details will be appended as query parameters. This URL must be in the server's list of allowed redirect URIs.\n * ``redirect_to`` (string, optional): A URL to redirect to *after* the server has successfully queued the email for sending (before the user clicks the link). If provided, a JSON response will not be returned, and parameters like ``email_sent`` (or ``code=true`` in Code method) will be appended as query parameters. This URL must be in the server's list of allowed redirect URIs.\n * ``link_url`` (string, optional): The base URL for the magic link itself (the endpoint the link in the email will point to). If not provided, it defaults to ``/magic-link/authenticate``. This URL must be in the server's list of allowed redirect URIs.\n\n**Response:**\n\nThe endpoint attempts to prevent email enumeration by always returning a success status if the request format is valid.\n\n- **Code method**\n\n * If the request accepts ``application/json`` and ``redirect_to`` is not provided, a 200 OK JSON response is returned:\n\n .. code-block:: json\n\n {\n \"code\": \"true\",\n \"signup\": \"true\",\n \"email\": \"user@example.com\"\n }\n\n * If ``redirect_to`` is provided, a 302 Found redirect occurs to ``redirect_to`` with ``code=true``, ``signup=true`` and ``email`` as query parameters.\n\n- **Link method**\n\n * If the request accepts ``application/json`` and ``redirect_to`` is not provided, a 200 OK JSON response is returned:\n\n .. code-block:: json\n\n {\n \"email_sent\": \"user@example.com\"\n }\n\n * If ``redirect_to`` is provided, a 302 Found redirect occurs to ``redirect_to`` with ``email_sent`` as a query parameter.\n\n- **Failure**\n\n * If an error occurs before a redirect would occur and the request accepts JSON, an HTTP error response (e.g., 400 Bad Request) is returned with a JSON body.\n * Otherwise, if ``redirect_on_failure`` was provided (Link method), a 302 Found redirect occurs to that URL with ``error`` and ``email`` query parameters.\n\n**Common Error Scenarios (leading to failure responses):**\n\n* Missing required fields in the request body: ``provider``, ``email``, ``challenge``, ``callback_url``, or ``redirect_on_failure``.\n* The provided ``callback_url``, ``redirect_on_failure``, ``redirect_to``, or ``link_url`` is not in the server's list of allowed redirect URIs.\n* Unsupported ``provider`` name.\n* Internal server error during email dispatch (e.g., SMTP issues).\n\nPOST /magic-link/email\n----------------------\n\nSends a magic link email to a user with an *existing* magic link credential. This is similar to ``POST /magic-link/register`` but does not attempt to create a new identity if the email is not found (though it still simulates a successful send to prevent enumeration).\n\n**Request Body (JSON or application/x-www-form-urlencoded):**\n\nThe required fields depend on the provider's verification method.\n\n- **Code method**\n\n * ``email`` (string, required): The user's email address.\n * ``redirect_to`` (string, optional): A URL to redirect to after the email has been queued. If omitted, the response will be JSON.\n\n- **Link method**\n\n * ``email`` (string, required): The user's email address.\n * ``challenge`` (string, required): A PKCE code challenge that will be embedded in the magic link token.\n * ``callback_url`` (string, required): The URL that the user will be redirected to after clicking the magic link in the email. A PKCE authorization ``code`` will be appended to this URL. This URL must be in the server's list of allowed redirect URIs.\n * ``redirect_on_failure`` (string, required): A URL to redirect to if there's an error during the email sending process. Error details will be appended as query parameters. This URL must be in the server's list of allowed redirect URIs.\n * ``redirect_to`` (string, optional): A URL to redirect to *after* the server has successfully queued the email for sending (before the user clicks the link). If provided, a JSON response will not be returned.\n * ``link_url`` (string, optional): The base URL for the magic link itself. If not provided, it defaults to ``/magic-link/authenticate``. This URL must be in the server's list of allowed redirect URIs.\n\n**Response:**\n\nThe endpoint attempts to prevent email enumeration by always returning a success status if the request format is valid, even if the email address is not found.\n\n- **Code method**\n\n * If ``redirect_to`` is NOT provided, a 200 OK JSON response is returned:\n\n .. code-block:: json\n\n {\n \"code\": \"true\",\n \"email\": \"user@example.com\"\n }\n\n * If ``redirect_to`` is provided, a 302 Found redirect occurs to the ``redirect_to`` URL with ``code=true`` and ``email`` as query parameters.\n\n- **Link method**\n\n * If ``redirect_to`` is NOT provided, a 200 OK JSON response is returned:\n\n .. code-block:: json\n\n {\n \"email_sent\": \"user@example.com\"\n }\n\n * If ``redirect_to`` is provided, a 302 Found redirect occurs to the ``redirect_to`` URL with ``email_sent`` as a query parameter.\n\n- **Failure**\n\n * If an error happens and a ``redirect_on_failure`` URL was provided (Link method), a 302 Found redirect is returned to that URL with ``error`` and the submitted ``email`` as query parameters. Otherwise, an HTTP error response is returned with a JSON body.\n\n**Common Error Scenarios (leading to failure responses):**\n\n* Missing required fields in the request body: ``provider``, ``email``, ``challenge``, ``callback_url``, or ``redirect_on_failure``.\n* The provided ``callback_url``, ``redirect_on_failure``, ``redirect_to``, or ``link_url`` is not in the server's list of allowed redirect URIs.\n* Unsupported ``provider`` name.\n* Internal server error during email dispatch (e.g., SMTP issues).\n\nPOST /magic-link/authenticate\n-----------------------------\n\nAuthenticates a user by validating a magic link token received from an email. This endpoint is typically the target of the magic link URL sent to the user.\n\nThis endpoint supports both Link and Code methods.\n\n**Link method (Query String):**\n\n* ``token`` (string, required): The magic link token (a signed JWT) extracted from the magic link URL. This token contains the identity ID, the original PKCE challenge, and the callback URL.\n* ``redirect_on_failure`` (string, optional): A URL to redirect to if the authentication process fails (e.g., invalid or expired token). Error details will be appended as query parameters. If not provided, an HTTP error response will be returned on failure.\n\n**Code method (JSON body):**\n\n* ``email`` (string, required): The user's email address.\n* ``code`` (string, required): The one-time code sent via email.\n* ``callback_url`` (string, required): The URL to redirect to after successful authentication. Must be an allowed redirect URI.\n* ``challenge`` (string, required): A PKCE code challenge. A PKCE authorization ``code`` will be generated upon success.\n\n**Response:**\n\n- **Link method**\n\n * If the provided ``token`` is valid, the user's email factor is marked as verified and a PKCE authorization ``code`` is generated using the challenge embedded in the token. A 302 Found redirect is returned to the token's ``callback_url`` with ``code`` appended.\n * On failure, if ``redirect_on_failure`` is provided, a 302 redirect occurs to that URL with an ``error`` parameter; otherwise, an HTTP error response is returned with a JSON body.\n\n- **Code method**\n\n * On success, the one-time code is validated, the email factor is marked as verified, and a PKCE authorization ``code`` is generated using the provided ``challenge``. A 302 Found redirect occurs to ``callback_url`` with ``code`` appended.\n * On failure, if a ``redirect_on_failure`` query parameter is present, a 302 redirect occurs to that URL with an ``error`` parameter; otherwise, a 400 Bad Request JSON response is returned with an error body.\n\n**Common Error Scenarios (leading to failure responses):**\n\n* Missing required ``token`` query parameter.\n* The provided ``token`` is malformed, has an invalid signature, or is expired.\n* Internal server error during the authentication or email verification process.\n* The ``callback_url`` extracted from the token is not in the server's list of allowed redirect URIs (this should ideally be caught earlier, but could potentially manifest here).\n" }, { "path": "docs/reference/auth/index.rst", "content": ".. _ref_guide_auth:\n\n====\nAuth\n====\n\n.. toctree::\n :hidden:\n :maxdepth: 3\n\n http\n built_in_ui\n email_password\n oauth\n magic_link\n webauthn\n webhooks\n\n:edb-alt-title: Using Gel Auth\n\n|Gel| Auth is a batteries-included authentication solution for your app built\ninto the Gel server. Here's how you can integrate it with your app.\n\n\nEnable extension in your schema\n===============================\n\nAuth is a Gel extension. To enable it, you will need to add the extension\nto your app's schema:\n\n.. code-block:: sdl\n\n using extension auth;\n\n\nExtension configuration\n=======================\n\nThe best and easiest way to configure the extension for your database is\nto use the built-in UI. To access it, run :gelcmd:`ui`. If you have the\nextension enabled in your schema as shown above and have migrated that\nschema change, you will see the \"Auth Admin\" icon in the left-hand toolbar.\n\n.. image:: images/ui-auth.png\n :alt: The Gel local development server UI highlighting the auth admin\n icon in the left-hand toolbar. The icon is two nested shield\n outlines, the inner being a light pink color and the outer being\n a light blue when selected.\n :width: 100%\n\nThe auth admin UI exposes these values:\n\n\napp_name\n--------\n\nThe name of your application to be shown on the login screen when using the\nbuilt-in UI.\n\n\nlogo_url\n--------\n\nA URL to an image of your logo. This is also used to customize the built-in UI.\n\n\nlogo_url\n--------\n\nA URL to an image of your logo for use with a dark theme. This is also used to\ncustomize the built-in UI.\n\n\nbrand_color\n-----------\n\nYour brand color as a hex string. This will be used as the accent color in the\nbuilt-in UI.\n\n\nauth_signing_key\n----------------\n\nThe extension uses JSON Web Tokens (JWTs) internally for many operations.\n``auth_signing_key`` is the value that is used as a symmetric key for signing\nthe JWTs. At the moment, the JWTs are not considered \"public\" API, so there is\nno need to save this value for your own application use. It is exposed mainly\nto allow rotation.\n\nTo configure via query or script:\n\n.. lint-off\n\n.. code-block:: edgeql\n\n CONFIGURE CURRENT BRANCH SET\n ext::auth::AuthConfig::auth_signing_key := 'F2KHaJfHi9Dzd8+6DI7FB9IFIoJXnhz2rzG/UzCRE7jTtYxqgTHHydc8xnN6emDB3tlR99FvPsyJfcVLVcQ5odSQpceDXplBOP+N14+EBy2mV6rA/7W7azIEKebtr9TVKrpBTMTOLAXo08ZnA6lvjn0VMs95za6Pta7VW62hjcb8jy6yxulvvU5SWnwa0x2z401K0pLK7byDD5eNqgTl40YaeOGoQ0iCkSmGxvLxyQgCIz2IU0zUbBwC9bQsTDORvflunruJznHuMxwbfYo/czQIIGuawU0H+G3GJZ3hecZLQlvwYCyLF37PFQVrcNMtUuGyDy2OyYtYHru2GW5B7Q';\n\n.. lint-on\n\ntoken_time_to_live\n------------------\n\nThis value controls the expiration time on the authentication token's\nJSON Web Token. This is effectively the \"session\" time.\n\nTo configure via query or script:\n\n.. code-block:: edgeql\n\n CONFIGURE CURRENT BRANCH SET\n ext::auth::AuthConfig::token_time_to_live := \"336 hours\";\n\nallowed_redirect_urls\n---------------------\n\nThis value is a set of strings that we use to ensure we only redirect to\ndomains that are under the control of the application using the Auth extension.\nWe compare any ``redirect_to`` URLs against this list. A URL is considered a\n\"match\" if the URL is exactly the same as one on the list, or is a sub-path of\na URL on the list.\n\nFor example, if the set includes ``https://example.com/myapp``:\n\n.. list-table::\n :header-rows: 1\n\n * - URL\n - Match\n * - ``https://example.com/myapp``\n - āœ…\n * - ``https://example.com/myapp/auth``\n - āœ…\n * - ``https://example.com/myapp/auth/verify``\n - āœ…\n * - ``https://example.com/myapp/somewhere/else``\n - āœ…\n * - ``http://example.com/myapp``\n - Does not match the protocol\n * - ``https://example.com:443/myapp``\n - Does not match the port\n * - ``https://auth.example.com/myapp``\n - Does not match the subdomain\n * - ``https://example.com/different/subpath``\n - Does not match the pathname or extend it\n\n.. note::\n\n šŸ’” We always allow redirects to the auth extension itself, so you do not\n need to add it explicitly if, for instance, you are always using the\n built-in UI.\n\n\nTo configure via query or script:\n\n.. code-block:: edgeql\n\n CONFIGURE CURRENT BRANCH SET\n ext::auth::AuthConfig::allowed_redirect_urls := {\n 'https://example.com',\n 'https://example.com/auth',\n 'https://localhost:3000',\n 'https://localhost:3000/auth'\n };\n\n\nWebhooks\n========\n\nThe auth extension supports sending webhooks for a variety of auth events. You can use these webhooks to, for instance, send a fully customized email for email verification, or password reset instead of our built-in email verification and password reset emails. You could also use them to trigger analytics events, start an email drip campaign, create an audit log, or trigger other side effects in your application.\n\nSee the :ref:`webhooks documentation ` for more details on how to configure and use webhooks.\n\nConfiguring SMTP\n================\n\n\nFor email-based factors, you can configure SMTP to allow the extension to send emails on your behalf. You should either configure SMTP, or webhooks for the relevant events.\n\nThe easiest way to configure SMTP is to use the built-in UI. Here is an example of configuring SMTP for local development using EdgeQL directly, using something like `Mailpit `__.\n\n.. note:: Gel Cloud users, rejoice!\n\n If you are using Gel Cloud, you can use the built-in development SMTP provider without any configuration. This special provider is already configured for development usage and is ready to send emails while you are developing your application. This provider is tuned specifically for development: it is rate limited and the sender is hardcoded. Do not use it in production, it will not work for that purpose.\n\n.. code-block:: edgeql\n\n # Create a new SMTP provider:\n #\n configure current branch\n insert cfg::SMTPProviderConfig {\n # This name must be unique and is used to reference the provider\n name := 'local_mailpit',\n sender := '\"Display Name\" ',\n host := 'localhost',\n port := 1025,\n username := 'smtpuser',\n password := 'smtppassword',\n security := 'STARTTLSOrPlainText',\n validate_certs := false,\n timeout_per_email := '60 seconds',\n timeout_per_attempt := '15 seconds',\n };\n\n # Set this provider as the current email provider by name:\n #\n configure current branch\n set current_email_provider_name := 'local_mailpit';\n\n.. note::\n\n The ``sender`` property follows the `RFC 5322 `_ specification, so you can include a display name in the email address or use a bare email address. Including a display name is recommended as it provides a more user-friendly experience. Email clients will show the display name (e.g., \"Display Name\" from the example above) instead of just the raw email address in the sender field, making your emails appear more professional and trustworthy to recipients.\n\nEnabling authentication providers\n=================================\n\nIn order to use the auth extension, you'll need to enable at least one of these\nauthentication providers. Providers can be added from the \"Providers\" section\nof the admin auth UI by clicking \"Add Provider.\" This will add a form to the UI\nallowing for selection of the provider and configuration of the values\ndescribed below.\n\nYou can also enable providers via query. We'll demonstrate how in each section\nbelow.\n\n\n.. _ref_guide_auth_overview_email_password:\n\nEmail and password\n------------------\n\n- ``require_verification``: (Default: ``true``) If ``true``, your application\n will not be able to retrieve an authentication token until the user\n has verified their email. If ``false``, your application can retrieve an\n authentication token, but a verification email will still be sent.\n Regardless of this setting, you can always decide to limit access or\n specific features in your application by testing if\n ``ext::auth::EmailPasswordFactor.verified_at`` is set to a date in\n the past on the ``ext::auth::LocalIdentity``.\n\nTo enable via query or script:\n\n.. code-block:: edgeql\n\n CONFIGURE CURRENT BRANCH\n INSERT ext::auth::EmailPasswordProviderConfig {\n require_verification := false,\n };\n\n.. note::\n\n ``require_verification`` defaults to ``true``.\n\nIf you use the Email and Password provider, in addition to the\n``require_verification`` configuration, you'll need to configure SMTP to allow\n|Gel| to send email verification and password reset emails on your behalf or\nset up webhooks for the relevant events:\n\n- ``ext::auth::WebhookEvent.EmailVerificationRequested``\n- ``ext::auth::WebhookEvent.PasswordResetRequested``\n\nHere is an example of setting a local SMTP server, in this case using a\nproduct called `Mailpit `__ which is\ngreat for testing in development:\n\n.. code-block:: edgeql\n\n CONFIGURE CURRENT BRANCH INSERT cfg::SMTPProviderConfig {\n sender := 'hello@example.com',\n host := 'localhost',\n port := 1025,\n security := 'STARTTLSOrPlainText',\n validate_certs := false,\n };\n\nHere is an example of setting up webhooks for the email verification and\npassword reset events:\n\n.. code-block:: edgeql\n\n CONFIGURE CURRENT BRANCH INSERT ext::auth::WebhookConfig {\n url := 'https://example.com/auth/webhook',\n events := {\n ext::auth::WebhookEvent.EmailVerificationRequested,\n ext::auth::WebhookEvent.PasswordResetRequested,\n }\n };\n\n\nOAuth\n-----\n\nWe currently support six different OAuth providers:\n\n.. lint-off\n\n- `Apple `__\n- `Azure\n (Microsoft) `__\n- `GitHub `__\n- `Google `__\n- `Discord `__\n- `Slack `__\n\n.. lint-on\n\nThe instructions for creating an app for each provider can be found on\neach provider's developer documentation website, which is linked above.\nThe important things you'll need to find and make note of for your\nconfiguration are the **client ID** and **secret**.\n\nOnce you select the OAuth provider in the configuration UI, you will need to\nprovide those values and the ``additional_scope``:\n\n- ``client_id`` This is assigned to you by the Identity Provider when\n you create an app with them.\n- ``secret`` This is created by the Identity Provider when you create\n an app with them.\n- ``additional_scope`` We request certain scope from the Identity\n Provider to fulfill our minimal data needs. You can pass additional\n scope here in a space-separated string and we will request that\n additional scope when getting the authentication token from the\n Identity Provider.\n\n .. note::\n\n We return this authentication token with this scope from the Identity\n Provider when we return our own authentication token.\n\nYou'll also need to set a callback URL in each provider's interface. To build\nthis callback URL, you will need the hostname, port, and branch name of your\ndatabase. The branch name is |main| by default. The hostname and port can\nbe found running this CLI command:\n\n.. code-block:: bash\n\n $ gel instance credentials\n\nThis will output a table that includes the hostnames and ports of all your\ninstances. Grab those from the row corresponding to the correct instance for\nuse in your callback URL, which takes on this format:\n\n.. code-block::\n\n http[s]://{gel_host}[:port]/db/{db_name}/ext/auth/callback\n\nTo enable the Azure OAuth provider via query or script:\n\n.. code-block:: edgeql\n\n CONFIGURE CURRENT BRANCH\n INSERT ext::auth::AzureOAuthProvider {\n secret := 'cccccccccccccccccccccccccccccccc',\n client_id := '1597b3fc-b67d-4d2b-b38f-acc256341dbc',\n additional_scope := 'offline_access',\n };\n\nTo enable any of the others, change ``AzureOAuthProvider`` in the example above\nto one of the other providers:\n\n- ``AppleOAuthProvider``\n- ``DiscordOAuthProvider``\n- ``GitHubOAuthProvider``\n- ``GoogleOAuthProvider``\n- ``SlackOAuthProvider``\n\n\nGeneric OpenID Connect providers\n--------------------------------\n\n.. versionadded:: 6.0\n\nGeneric OpenID Connect providers are now supported. In order to use them,\nyou will need to insert an ``ext::auth::OpenIDConnectProvider`` configuration\nobject with a few additional properties:\n\n- ``name``: A unique string identifying the provider.\n- ``display_name``: A human-readable name for the provider.\n- ``issuer_url``: The issuer URL of the provider. This must be the domain of\n the provider's authorization server and will be used to drive the OpenID\n Connect flow.\n- ``logo_url``: (optional) A URL to an image of the provider's logo. This is\n used in the built-in UI to display the correct logo.\n\nInherited from ``ext::auth::OAuthProviderConfig``:\n\n- ``client_id``: The client ID of the provider.\n- ``secret``: The client secret of the provider.\n- ``additional_scope``: (optional) A space-separated string of additional\n scopes to request from the provider.\n\nHere is an example of enabling the Google OpenID Connect provider (note, for\nGoogle, you can simply use the existing Google provider, but this is for\nillustration purposes):\n\n.. lint-off\n\n.. code-block:: edgeql\n\n CONFIGURE CURRENT BRANCH\n INSERT ext::auth::OpenIDConnectProvider {\n name := 'google',\n display_name := 'Google',\n issuer_url := 'https://accounts.google.com',\n logo_url := 'https://www.google.com/images/branding/googlelogo/1x/googlelogo_color_272x92dp.png',\n client_id := '1234567890',\n secret := '1234567890',\n };\n\n.. lint-on\n\nMagic link\n----------\n\nMagic link offers the following settings:\n\n- ``verification_method``: ``Link`` (default) or ``Code``.\n - ``Link``: users receive a link and are redirected back with a PKCE ``code``.\n - ``Code``: users receive a one-time code. Collect the code and call ``POST /magic-link/authenticate`` with ``email``, ``code``, ``callback_url``, and the PKCE ``challenge`` to receive a PKCE ``code``.\n\n- ``token_time_to_live``: determines how long a magic link (or one-time code) remains valid after sending.\n\nSince magic links rely on email, you must also configure SMTP or webhooks. For\nlocal testing, you can use the same method used for SMTP previously for\n:ref:`the email and password provider `.\n\nHere is an example of setting a local SMTP server, in this case using a\nproduct called `Mailpit `__ which is\ngreat for testing in development:\n\n.. code-block:: edgeql\n\n CONFIGURE CURRENT BRANCH INSERT cfg::SMTPProviderConfig {\n sender := 'hello@example.com',\n host := 'localhost',\n port := 1025,\n security := 'STARTTLSOrPlainText',\n validate_certs := false,\n };\n\nHere is an example of setting up webhooks for the magic link events:\n\n.. code-block:: edgeql\n\n CONFIGURE CURRENT BRANCH INSERT ext::auth::WebhookConfig {\n url := 'https://example.com/auth/webhook',\n events := {\n ext::auth::WebhookEvent.MagicLinkRequested,\n }\n };\n\n\nWebAuthn\n--------\n\n- ``relying_party_origin``: This is the URL of the web application handling\n the WebAuthn request. If you're using the built-in UI, it's the origin of\n the Gel web server.\n\n- ``require_verification``: (Default: ``true``) If ``true``, your application\n will not be able to retrieve an authentication token until the user has\n verified their email. If ``false``, your application can retrieve an\n authentication token, but a verification email will still be sent.\n Regardless of this setting, you can always decide to limit access or\n specific features in your application by testing if\n ``ext::auth::WebAuthnFactor.verified_at`` is set to a date in the past on\n the ``ext::auth::LocalIdentity``.\n\n.. note::\n\n You will need to configure SMTP or webhooks. For local testing, you can use\n Mailpit as described in :ref:`the email/password section\n `.\n\n.. note::\n\n You will need to configure CORS to allow the client-side script to call the\n Gel Auth extension's endpoints from the web browser. You can do this by\n updating the ``cors_allow_origins`` configuration in the Gel server\n configuration.\n\nHere is an example of setting a local SMTP server, in this case using a\nproduct called `Mailpit `__ which is\ngreat for testing in development:\n\n.. code-block:: edgeql\n\n CONFIGURE CURRENT BRANCH INSERT cfg::SMTPProviderConfig {\n sender := 'hello@example.com',\n host := 'localhost',\n port := 1025,\n security := 'STARTTLSOrPlainText',\n validate_certs := false,\n };\n\nHere is an example of setting up webhooks for the WebAuthn events:\n\n.. code-block:: edgeql\n\n CONFIGURE CURRENT BRANCH INSERT ext::auth::WebhookConfig {\n url := 'https://example.com/auth/webhook',\n events := {\n ext::auth::WebhookEvent.EmailVerificationRequested,\n }\n };\n\n\nIntegrating your application\n============================\n\nIn the end, what we want to end up with is an authentication token\ncreated by Gel that we can set as a global in any authenticated\nqueries executed from our application, which will set a computed global linked\nto an ``ext::auth::Identity``.\n\n.. note::\n\n šŸ’” If you want your own ``User`` type that contains application specific\n information like name, preferences, etc, you can link to this\n ``ext::auth::Identity`` to do so.\n\nYou can then use the ``ext::auth::Identity`` (or custom ``User`` type)\nto define access policies and make authenticated queries.\n\nSelect your method for detailed configuration:\n\n.. toctree::\n :maxdepth: 3\n\n built_in_ui\n email_password\n oauth\n magic_link\n webauthn\n\n\nExample usage\n=============\n\nHere's an example schema that we can use to show how you would use the\n``auth_token`` you get back from Gel to make queries against a\nprotected resource, in this case being able to insert a ``Post``.\n\n.. code-block:: sdl\n\n using extension auth;\n\n module default {\n global current_user := (\n assert_single((\n select User\n filter .identity = global ext::auth::ClientTokenIdentity\n ))\n );\n\n type User {\n required name: str;\n required identity: ext::auth::Identity;\n }\n\n type Post {\n required text: str;\n required author: User;\n\n access policy author_has_full_access\n allow all\n using (.author ?= global current_user);\n\n access policy others_read_only\n allow select;\n }\n }\n\nLet's now insert a ``Post``.\n\n.. lint-off\n\n.. code-block:: tsx\n\n const client = createClient().withGlobals({\n \"ext::auth::client_token\": auth_token,\n });\n\n const inserted = await client.querySingle(\n `\n insert Post {\n text := $text,\n author := global current_user,\n }`,\n {\n text: 'if your grave doesnt say \"rest in peace\" on it you are automatically drafted into the skeleton war'\n }\n );\n\n.. lint-on\n\nI can even delete it, since I have access through the global:\n\n.. code-block:: tsx\n\n await client.query(`delete Post filter .id = $id`, {\n id: inserted.id\n });\n" }, { "path": "docs/reference/auth/magic_link.rst", "content": ".. _ref_guide_auth_magic_link:\n\n================\nMagic Link Auth\n================\n\n:edb-alt-title: Integrating Gel Auth's Magic Link provider\n\nMagic Link is a passwordless authentication method that allows users to log in via a unique, time-sensitive link sent to their email. This guide will walk you through integrating Magic Link authentication with your application using Gel Auth.\n\nEnable Magic Link provider\n==========================\n\nBefore you can use Magic Link authentication, you need to enable the Magic Link provider in your Gel Auth configuration. This can be done through the Gel UI under the \"Providers\" section.\n\nMagic Link flow\n===============\n\nThe Magic Link authentication flow involves three main steps:\n\n1. **Sending a Magic Link Email**: Your application requests Gel Auth to send a magic link to the user's email.\n\n2. **User Clicks Magic Link**: The user receives the email and clicks on the magic link.\n\n3. **Authentication and Token Retrieval**: The magic link directs the user to your application, which then authenticates the user and retrieves an authentication token from Gel Auth.\n\n.. note::\n\n Your Magic Link provider can be configured to send either a **Link** or a one-time **Code**:\n\n - **Code**: Instead of clicking a link, users receive a one-time code. Prompt for the code and call ``POST /magic-link/authenticate`` with ``email``, ``code``, ``callback_url`` and the PKCE ``challenge``. On success, you will be redirected to ``callback_url`` with a PKCE ``code`` you can exchange at ``POST /token``.\n - **Link**: Users click a link that includes a token. Your app handles the token-based callback as shown below.\n\nUI considerations\n=================\n\nSimilar to how the built-in UI works, you can query the database configuration\nto discover which providers are configured and dynamically build the UI.\n\n.. code-block:: edgeql\n\n select cfg::Config.extensions[is ext::auth::AuthConfig].providers {\n name,\n [is ext::auth::OAuthProviderConfig].display_name,\n };\n\nThe ``name`` is a unique string that identifies the Identity Provider. OAuth\nproviders also have a ``display_name`` that you can use as a label for links or\nbuttons.\n\n\nExample implementation\n======================\n\nWe will demonstrate the various steps below by building a NodeJS HTTP server in\na single file that we will use to simulate a typical web application.\n\n.. note::\n\n The details below show the inner workings of how data is exchanged with the\n Auth extension from a web app using HTTP. You can use this as a guide to\n integrate with your application written in any language that can send and\n receive HTTP requests.\n\n\nStart the PKCE flow\n-------------------\n\nWe secure authentication tokens and other sensitive data by using PKCE\n(Proof Key of Code Exchange).\n\nYour application server creates a 32-byte Base64 URL-encoded string (which will\nbe 43 bytes after encoding), called the ``verifier``. You need to store this\nvalue for the duration of the flow. One way to accomplish this bit of state is\nto use an HttpOnly cookie when the browser makes a request to the server for\nthis value, which you can then use to retrieve it from the cookie store at the\nend of the flow. Take this ``verifier`` string, hash it with SHA256, and then\nbase64url encode the resulting string. This new string is called the\n``challenge``.\n\n.. lint-off\n\n.. code-block:: javascript\n\n import http from \"node:http\";\n import { URL } from \"node:url\";\n import crypto from \"node:crypto\";\n\n /**\n * You can get this value by running `gel instance credentials`.\n * Value should be:\n * `${protocol}://${host}:${port}/branch/${branch}/ext/auth/\n */\n const GEL_AUTH_BASE_URL = process.env.GEL_AUTH_BASE_URL;\n const SERVER_PORT = 3000;\n\n /**\n * Generate a random Base64 url-encoded string, and derive a \"challenge\"\n * string from that string to use as proof that the request for a token\n * later is made from the same user agent that made the original request\n *\n * @returns {Object} The verifier and challenge strings\n */\n const generatePKCE = () => {\n const verifier = crypto.randomBytes(32).toString(\"base64url\");\n\n const challenge = crypto\n .createHash(\"sha256\")\n .update(verifier)\n .digest(\"base64url\");\n\n return { verifier, challenge };\n };\n\n.. lint-on\n\nRouting\n-------\n\nLet's set up the routes we will use to handle the magic link authentication\nflow. We will then detail each route handler in the following sections.\n\n.. lint-off\n\n.. code-block:: javascript\n\n const server = http.createServer(async (req, res) => {\n const requestUrl = getRequestUrl(req);\n\n switch (requestUrl.pathname) {\n case \"/auth/magic-link/callback\": {\n await handleCallback(req, res);\n break;\n }\n\n case \"/auth/magic-link/signup\": {\n await handleSignUp(req, res);\n break;\n }\n\n case \"/auth/magic-link/send\": {\n await handleSendMagicLink(req, res);\n break;\n }\n\n default: {\n res.writeHead(404);\n res.end(\"Not found\");\n break;\n }\n }\n });\n\n.. lint-on\n\nSign up\n-------\n\n.. lint-off\n\n.. code-block:: javascript\n\n /**\n * Send magic link to new user's email for sign up.\n *\n * @param {Request} req\n * @param {Response} res\n */\n const handleSignUp = async (req, res) => {\n let body = \"\";\n req.on(\"data\", (chunk) => {\n body += chunk.toString();\n });\n req.on(\"end\", async () => {\n const pkce = generatePKCE();\n const { email, provider } = JSON.parse(body);\n if (!email || !provider) {\n res.status = 400;\n res.end(\n `Request body malformed. Expected JSON body with 'email' and 'provider' keys, but got: ${body}`,\n );\n return;\n }\n\n const registerUrl = new URL(\"magic-link/register\", GEL_AUTH_BASE_URL);\n const callbackUrl = new URL(\"auth/magic-link/callback\", \"http://localhost:${SERVER_PORT}\");\n const registerResponse = await fetch(registerUrl.href, {\n method: \"post\",\n headers: {\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({\n challenge: pkce.challenge,\n email,\n provider,\n callback_url: callbackUrl.href,\n // The following endpoint will be called if there is an error\n // processing the magic link, such as expiration or malformed token,\n // etc.\n redirect_on_failure: `http://localhost:${SERVER_PORT}/auth_error.html`,\n }),\n });\n\n if (!registerResponse.ok) {\n const text = await registerResponse.text();\n res.status = 400;\n res.end(`Error from the auth server: ${text}`);\n return;\n }\n\n res.writeHead(204, {\n \"Set-Cookie\": `gel-pkce-verifier=${pkce.verifier}; HttpOnly; Path=/; Secure; SameSite=Strict`,\n });\n res.end();\n });\n };\n\n.. lint-on\n\nSign in\n-------\n\nSigning in with a magic link simply involves telling the Gel Auth server to\nsend a magic link to the user's email. The user will then click on the link to\nauthenticate.\n\n.. lint-off\n\n.. code-block:: javascript\n\n /**\n * Send magic link to existing user's email for sign in.\n *\n * @param {Request} req\n * @param {Response} res\n */\n const handleSendMagicLink = async (req, res) => {\n let body = \"\";\n req.on(\"data\", (chunk) => {\n body += chunk.toString();\n });\n req.on(\"end\", async () => {\n const pkce = generatePKCE();\n const { email, provider } = JSON.parse(body);\n if (!email || !provider) {\n res.status = 400;\n res.end(\n `Request body malformed. Expected JSON body with 'email' and 'provider' keys, but got: ${body}`,\n );\n return;\n }\n\n const emailUrl = new URL(\"magic-link/email\", GEL_AUTH_BASE_URL);\n const callbackUrl = new URL(\"auth/magic-link/callback\", \"http://localhost:${SERVER_PORT}\");\n const authenticateResponse = await fetch(emailUrl.href, {\n method: \"post\",\n headers: {\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({\n challenge: pkce.challenge,\n email,\n provider,\n callback_url: callbackUrl.href,\n }),\n });\n\n if (!authenticateResponse.ok) {\n const text = await authenticateResponse.text();\n res.status = 400;\n res.end(`Error from the auth server: ${text}`);\n return;\n }\n\n res.writeHead(204, {\n \"Set-Cookie\": `gel-pkce-verifier=${pkce.verifier}; HttpOnly; Path=/; Secure; SameSite=Strict`,\n });\n res.end();\n });\n };\n\n.. lint-on\n\nCallback\n--------\n\nOnce the user clicks on the magic link (Link method), they will be redirected\nback to your application with a ``code`` query parameter. Your application will\nthen exchange this code for an authentication token.\n\nIf the provider uses the Code method, you will instead collect the one-time\ncode from the user and call ``POST /magic-link/authenticate`` with\n``email``, ``code``, ``callback_url`` and the PKCE ``challenge``. On success\nyou will be redirected to ``callback_url`` with a PKCE ``code`` to exchange at\n``POST /token``.\n\n.. lint-off\n\n.. code-block:: javascript\n\n /**\n * Handles the PKCE callback and exchanges the `code` and `verifier`\n * for an auth_token, setting the auth_token as an HttpOnly cookie.\n *\n * @param {Request} req\n * @param {Response} res\n */\n const handleCallback = async (req, res) => {\n const requestUrl = getRequestUrl(req);\n\n const code = requestUrl.searchParams.get(\"code\");\n if (!code) {\n const error = requestUrl.searchParams.get(\"error\");\n res.status = 400;\n res.end(\n `Magic link callback is missing 'code'. Provider responded with error: ${error}`,\n );\n return;\n }\n\n const cookies = req.headers.cookie?.split(\"; \");\n const verifier = cookies\n ?.find((cookie) => cookie.startsWith(\"gel-pkce-verifier=\"))\n ?.split(\"=\")[1];\n if (!verifier) {\n res.status = 400;\n res.end(\n `Could not find 'verifier' in the cookie store. Is this the same user agent/browser that started the authorization flow?`,\n );\n return;\n }\n\n const codeExchangeUrl = new URL(\"token\", GEL_AUTH_BASE_URL);\n codeExchangeUrl.searchParams.set(\"code\", code);\n codeExchangeUrl.searchParams.set(\"verifier\", verifier);\n const codeExchangeResponse = await fetch(codeExchangeUrl.href, {\n method: \"GET\",\n });\n\n if (!codeExchangeResponse.ok) {\n const text = await codeExchangeResponse.text();\n res.status = 400;\n res.end(`Error from the auth server: ${text}`);\n return;\n }\n\n const { auth_token } = await codeExchangeResponse.json();\n res.writeHead(204, {\n \"Set-Cookie\": `gel-auth-token=${auth_token}; HttpOnly; Path=/; Secure; SameSite=Strict`,\n });\n res.end();\n };\n\n.. lint-on\n\n\nCreate a User object\n--------------------\n\nFor some applications, you may want to create a custom ``User`` type in the\ndefault module to attach application-specific information. You can tie this to\nan ``ext::auth::Identity`` by using the ``identity_id`` returned during the\nsign-up flow.\n\n.. note::\n\n For this example, we'll assume you have a one-to-one relationship between\n ``User`` objects and ``ext::auth::Identity`` objects. In your own\n application, you may instead decide to have a one-to-many relationship.\n\nGiven this ``User`` type:\n\n.. code-block:: sdl\n\n type User {\n email: str;\n name: str;\n\n required identity: ext::auth::Identity {\n constraint exclusive;\n };\n }\n\nWe need to update two parts of the sign-up flow. First, we need to signal to the callback that this particular callback is for a sign-up, which we do by setting the ``isSignUp`` query parameter to ``true``. Second, we need to create a new ``User`` object and attach it to the ``ext::auth::Identity`` object.\n\n.. tabs::\n\n .. code-tab:: javascript-diff\n :caption: handleSignUp\n\n const handleSignUp = async (req, res) => {\n let body = \"\";\n req.on(\"data\", (chunk) => {\n body += chunk.toString();\n });\n req.on(\"end\", async () => {\n const pkce = generatePKCE();\n const { email, provider } = JSON.parse(body);\n if (!email || !provider) {\n res.status = 400;\n res.end(\n `Request body malformed. Expected JSON body with 'email' and 'provider' keys, but got: ${body}`,\n );\n return;\n }\n\n const registerUrl = new URL(\"magic-link/register\", GEL_AUTH_BASE_URL);\n const callbackUrl = new URL(\"auth/magic-link/callback\", \"http://localhost:${SERVER_PORT}\");\n + callbackUrl.searchParams.set(\"isSignUp\", \"true\");\n const registerResponse = await fetch(registerUrl.href, {\n method: \"post\",\n headers: {\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({\n challenge: pkce.challenge,\n email,\n provider,\n callback_url: callbackUrl.href,\n // The following endpoint will be called if there is an error\n // processing the magic link, such as expiration or malformed token,\n // etc.\n redirect_on_failure: `http://localhost:${SERVER_PORT}/auth_error.html`,\n }),\n });\n\n if (!registerResponse.ok) {\n const text = await registerResponse.text();\n res.status = 400;\n res.end(`Error from the auth server: ${text}`);\n return;\n }\n\n res.writeHead(204, {\n \"Set-Cookie\": `gel-pkce-verifier=${pkce.verifier}; HttpOnly; Path=/; Secure; SameSite=Strict`,\n });\n res.end();\n });\n };\n\n .. code-tab:: javascript-diff\n :caption: handleCallback\n\n const handleCallback = async (req, res) => {\n const requestUrl = getRequestUrl(req);\n\n const code = requestUrl.searchParams.get(\"code\");\n if (!code) {\n const error = requestUrl.searchParams.get(\"error\");\n res.status = 400;\n res.end(\n `Magic link callback is missing 'code'. Provider responded with error: ${error}`,\n );\n return;\n }\n\n const cookies = req.headers.cookie?.split(\"; \");\n const verifier = cookies\n ?.find((cookie) => cookie.startsWith(\"gel-pkce-verifier=\"))\n ?.split(\"=\")[1];\n if (!verifier) {\n res.status = 400;\n res.end(\n `Could not find 'verifier' in the cookie store. Is this the same user agent/browser that started the authorization flow?`,\n );\n return;\n }\n\n const codeExchangeUrl = new URL(\"token\", GEL_AUTH_BASE_URL);\n codeExchangeUrl.searchParams.set(\"code\", code);\n codeExchangeUrl.searchParams.set(\"verifier\", verifier);\n const codeExchangeResponse = await fetch(codeExchangeUrl.href, {\n method: \"GET\",\n });\n\n if (!codeExchangeResponse.ok) {\n const text = await codeExchangeResponse.text();\n res.status = 400;\n res.end(`Error from the auth server: ${text}`);\n return;\n }\n\n - const { auth_token } = await codeExchangeResponse.json();\n + const {\n + auth_token,\n + identity_id\n + } = await codeExchangeResponse.json();\n\n + if (requestUrl.searchParams.get(\"isSignUp\") === \"true\") {\n + await client.query(`\n + with\n + identity := $identity_id,\n + emailFactor := (\n + select ext::auth::EmailFactor filter .identity = identity\n + ),\n + insert User {\n + email := emailFactor.email,\n + identity := identity\n + };\n + `, { identity_id });\n + }\n +\n res.writeHead(204, {\n \"Set-Cookie\": `gel-auth-token=${auth_token}; HttpOnly; Path=/; Secure; SameSite=Strict`,\n });\n res.end();\n };\n\n:ref:`Back to the Gel Auth guide `\n" }, { "path": "docs/reference/auth/oauth.rst", "content": ".. _ref_guide_auth_oauth:\n\n=====\nOAuth\n=====\n\n:edb-alt-title: Integrating Gel Auth's OAuth provider\n\nAlong with using the :ref:`built-in UI `, you can also\ncreate your own UI that calls to your own web application backend.\n\nUI considerations\n=================\n\nSimilar to how the built-in UI works, you can query the database configuration\nto discover which providers are configured and dynamically build the UI.\n\n.. code-block:: edgeql\n\n select cfg::Config.extensions[is ext::auth::AuthConfig].providers {\n name,\n [is ext::auth::OAuthProviderConfig].display_name,\n };\n\nThe ``name`` is a unique string that identifies the Identity Provider. OAuth\nproviders also have a ``display_name`` that you can use as a label for links or\nbuttons. In later steps, you'll be providing this ``name`` as the ``provider``\nin various endpoints.\n\n\nExample implementation\n======================\n\nWe will demonstrate the various steps below by building a NodeJS HTTP server in\na single file that we will use to simulate a typical web application.\n\n.. note::\n\n We are in the process of publishing helper libraries that you can use with\n popular languages and web frameworks. The details below show the inner\n workings of how data is exchanged with the Auth extension from a web app\n using HTTP. You can use this as a guide to integrate with your application\n written in any language that can send and receive HTTP requests.\n\nWe secure authentication tokens and other sensitive data by using PKCE\n(Proof Key of Code Exchange).\n\n\nStart the PKCE flow\n-------------------\n\nYour application server creates a 32-byte Base64 URL-encoded string (which will\nbe 43 bytes after encoding), called the ``verifier``. You need to store this\nvalue for the duration of the flow. One way to accomplish this bit of state is\nto use an HttpOnly cookie when the browser makes a request to the server for\nthis value, which you can then use to retrieve it from the cookie store at the\nend of the flow. Take this ``verifier`` string, hash it with SHA256, and then\nbase64url encode the resulting string. This new string is called the\n``challenge``.\n\n.. note::\n\n Since ``=`` is not a URL-safe character, if your Base64-URL encoding\n function adds padding, you should remove the padding before hashing the\n ``verifier`` to derive the ``challenge`` or when providing the ``verifier``\n or ``challenge`` in your requests.\n\n.. note::\n\n If you are familiar with PKCE, you will notice some differences from how RFC\n 7636 defines PKCE. Our authentication flow is not an OAuth flow, but rather a\n strict server-to-server flow with Proof Key of Code Exchange added for\n additional security to avoid leaking the authentication token. Here are some\n differences between PKCE as defined in RFC 7636 and our implementation:\n\n - We do not support the ``plain`` value for ``code_challenge_method``, and\n therefore do not read that value if provided in requests.\n - Our parameters omit the ``code_`` prefix, however we do support\n ``code_challenge`` and ``code_verifier`` as aliases, preferring\n ``challenge`` and ``verifier`` if present.\n\n.. code-block:: javascript\n\n import http from \"node:http\";\n import { URL } from \"node:url\";\n import crypto from \"node:crypto\";\n\n /**\n * You can get this value by running `gel instance credentials`.\n * Value should be:\n * `${protocol}://${host}:${port}/branch/${branch}/ext/auth/\n */\n const GEL_AUTH_BASE_URL = process.env.GEL_AUTH_BASE_URL;\n const SERVER_PORT = 3000;\n\n /**\n * Generate a random Base64 url-encoded string, and derive a \"challenge\"\n * string from that string to use as proof that the request for a token\n * later is made from the same user agent that made the original request\n *\n * @returns {Object} The verifier and challenge strings\n */\n const generatePKCE = () => {\n const verifier = crypto.randomBytes(32).toString(\"base64url\");\n\n const challenge = crypto\n .createHash(\"sha256\")\n .update(verifier)\n .digest(\"base64url\");\n\n return { verifier, challenge };\n };\n\n\n.. note::\n\n For |EdgeDB| versions before 5.0, the value for :gelenv:`AUTH_BASE_URL`\n in the above snippet should have the form:\n\n ``${protocol}://${host}:${port}/db/${database}/ext/auth/``\n\n\nRedirect users to Identity Provider\n-----------------------------------\n\nNext, we implement a route at ``/auth/authorize`` that the application should\nlink to when signing in with a particular Identity Provider. We will redirect\nthe end user's browser to the Identity Provider with the proper setup.\n\n.. lint-off\n\n.. code-block:: javascript\n\n const server = http.createServer(async (req, res) => {\n const requestUrl = getRequestUrl(req);\n\n switch (requestUrl.pathname) {\n case \"/auth/authorize\": {\n await handleAuthorize(req, res);\n break;\n }\n\n case \"/auth/callback\": {\n await handleCallback(req, res);\n break;\n }\n\n default: {\n res.writeHead(404);\n res.end(\"Not found\");\n break;\n }\n }\n });\n\n /**\n * Redirects OAuth requests to Gel Auth OAuth authorize redirect\n * with the PKCE challenge, and saves PKCE verifier in an HttpOnly\n * cookie for later retrieval.\n *\n * @param {Request} req\n * @param {Response} res\n */\n const handleAuthorize = async (req, res) => {\n const requestUrl = getRequestUrl(req);\n const provider = requestUrl.searchParams.get(\"provider\");\n\n if (!provider) {\n res.status = 400;\n res.end(\"Must provider a 'provider' value in search parameters\");\n return;\n }\n\n const pkce = generatePKCE();\n const redirectUrl = new URL(\"authorize\", GEL_AUTH_BASE_URL);\n redirectUrl.searchParams.set(\"provider\", provider);\n redirectUrl.searchParams.set(\"challenge\", pkce.challenge);\n redirectUrl.searchParams.set(\n \"redirect_to\",\n `http://localhost:${SERVER_PORT}/auth/callback`\n );\n redirectUrl.searchParams.set(\n \"redirect_to_on_signup\",\n `http://localhost:${SERVER_PORT}/auth/callback?isSignUp=true`\n );\n\n res.writeHead(302, {\n \"Set-Cookie\": `gel-pkce-verifier=${pkce.verifier}; HttpOnly; Path=/; Secure; SameSite=Strict`,\n Location: redirectUrl.href,\n });\n res.end();\n };\n\n.. lint-on\n\n\nRetrieve ``auth_token``\n-----------------------\n\nAt the very end of the flow, the Gel server will redirect the user's browser\nto the ``redirect_to`` address with a single query parameter: ``code``. This\nroute should be a server route that has access to the ``verifier``. You then\ntake that ``code`` and look up the ``verifier`` in the ``gel-pkce-verifier``\ncookie (``gel-pkce-verifier`` with |EdgeDB| <= 5), and make a request to\nthe Gel Auth extension to exchange these two pieces of data for an\n``auth_token``.\n\n.. lint-off\n\n.. code-block:: javascript\n\n /**\n * Handles the PKCE callback and exchanges the `code` and `verifier\n * for an auth_token, setting the auth_token as an HttpOnly cookie.\n *\n * @param {Request} req\n * @param {Response} res\n */\n const handleCallback = async (req, res) => {\n const requestUrl = getRequestUrl(req);\n\n const code = requestUrl.searchParams.get(\"code\");\n if (!code) {\n const error = requestUrl.searchParams.get(\"error\");\n res.status = 400;\n res.end(\n `OAuth callback is missing 'code'. OAuth provider responded with error: ${error}`\n );\n return;\n }\n\n const cookies = req.headers.cookie?.split(\"; \");\n const verifier = cookies\n ?.find((cookie) => cookie.startsWith(\"gel-pkce-verifier=\"))\n ?.split(\"=\")[1];\n if (!verifier) {\n res.status = 400;\n res.end(\n `Could not find 'verifier' in the cookie store. Is this the same user agent/browser that started the authorization flow?`\n );\n return;\n }\n\n const codeExchangeUrl = new URL(\"token\", GEL_AUTH_BASE_URL);\n codeExchangeUrl.searchParams.set(\"code\", code);\n codeExchangeUrl.searchParams.set(\"verifier\", verifier);\n const codeExchangeResponse = await fetch(codeExchangeUrl.href, {\n method: \"GET\",\n });\n\n if (!codeExchangeResponse.ok) {\n const text = await codeExchangeResponse.text();\n res.status = 400;\n res.end(`Error from the auth server: ${text}`);\n return;\n }\n\n const { auth_token } = await codeExchangeResponse.json();\n res.writeHead(204, {\n \"Set-Cookie\": `gel-auth-token=${auth_token}; HttpOnly; Path=/; Secure; SameSite=Strict`,\n });\n res.end();\n };\n\n.. lint-on\n\nCreating a User object\n----------------------\n\nFor some applications, you may want to create a custom ``User`` type in the\ndefault module to attach application-specific information. You can tie this to\nan ``ext::auth::Identity`` by using the ``auth_token`` in our\n``ext::auth::client_token`` global and inserting your ``User`` object with a\nlink to the ``Identity``.\n\n.. note::\n\n For this example, we'll assume you have a one-to-one relationship between\n ``User`` objects and ``ext::auth::Identity`` objects. In your own\n application, you may instead decide to have a one-to-many relationship.\n\nGiven this ``User`` type:\n\n.. code-block:: sdl\n\n type User {\n email: str;\n name: str;\n\n required identity: ext::auth::Identity {\n constraint exclusive;\n };\n }\n\nYou can update the callback function like this to create a new ``User`` object\nwhen the callback succeeds. Recall that in our ``handleAuthorize`` route\nhandler, we added a separate callback route for when the extension adds a new\nIdentity which sets a search parameter on the URL to ``isSignUp=true``:\n\n.. code-block:: javascript-diff\n\n const { auth_token } = await codeExchangeResponse.json();\n +\n + const isSignUp = requestUrl.searchParams.get(\"isSignUp\");\n + if (isSignUp === \"true\") {\n + const authedClient = client.withGlobals({\n + \"ext::auth::client_token\": auth_token,\n + });\n + await authedClient.query(`\n + insert User {\n + identity := (global ext::auth::ClientTokenIdentity)\n + };\n + `);\n + }\n +\n res.writeHead(204, {\n \"Set-Cookie\": `gel-auth-token=${auth_token}; HttpOnly; Path=/; Secure; SameSite=Strict`,\n });\n\n\nUsing an OpenID Connect id_token\n--------------------------------\n\nFor some providers that implement OpenID Connect, we also return an ``id_token``\nin the response. This token will have been validated by the extension to\nensure that it has been signed by the provider and that the token has not\nexpired. You can use this token to get additional information about the user\nfrom the provider to enrich your ``User`` object.\n\n.. code-block:: javascript-diff\n\n - const { auth_token } = await codeExchangeResponse.json();\n + const { auth_token, id_token } = await codeExchangeResponse.json();\n\n const isSignUp = requestUrl.searchParams.get(\"isSignUp\");\n if (isSignUp === \"true\") {\n + const { email, name, locale } = id_token\n ? await decodeJwt(id_token)\n : { email: null, name: null, locale: null };\n const authedClient = client.withGlobals({\n \"ext::auth::client_token\": auth_token,\n });\n await authedClient.query(`\n insert User {\n identity := (global ext::auth::ClientTokenIdentity)\n + email := email,\n + name := name,\n + locale := locale\n };\n - `);\n + `, { email, name, locale });\n }\n\n\nMaking authenticated requests to the OAuth resource server\n----------------------------------------------------------\n\nAlong with the ``auth_token`` which represents the authenticated user's\nidentity within your system, for OAuth providers, we also return a\n``provider_token`` (and optionally a ``provider_refresh_token``) that you can\nuse to make requests to the OAuth provider's resource server on behalf of the\nuser.\n\nHere is an example of getting the user's profile information from Google\nutilizing OpenID Connect and the ``provider_token``:\n\n.. code-block:: javascript\n\n /**\n * Get the user's profile information from Google\n */\n async function getUserProfile(providerToken) {\n const response = await fetch(\n \"https://accounts.google.com/.well-known/openid-configuration\"\n );\n const discoveryDocument = await response.json();\n const response = await fetch(discoveryDocument.userinfo_endpoint, {\n headers: {\n Authorization: `Bearer ${providerToken}`,\n Accept: \"application/json\",\n },\n });\n return await response.json();\n }\n\nThen in our callback handler, we can use the ``provider_token`` to get the\nuser's profile information and save it into our ``User`` object when we create\nit:\n\n.. code-block:: javascript-diff\n\n - const { auth_token } = await codeExchangeResponse.json();\n + const { auth_token, provider_token } = await codeExchangeResponse.json();\n\n const isSignUp = requestUrl.searchParams.get(\"isSignUp\");\n if (isSignUp === \"true\") {\n + const profile = await getUserProfile(provider_token);\n const authedClient = client.withGlobals({\n \"ext::auth::client_token\": auth_token,\n });\n await authedClient.query(\n `\n + with\n + email := $email,\n + name := $name,\n insert User {\n + email := email,\n + name := name,\n identity := (global ext::auth::ClientTokenIdentity)\n };\n - `);\n + `,\n + { email: profile.email, name: profile.name }\n + );\n }\n\n res.writeHead(204, {\n \"Set-Cookie\": `gel-auth-token=${auth_token}; HttpOnly; Path=/; Secure; SameSite=Strict`,\n });\n\n:ref:`Back to the Gel Auth guide `\n" }, { "path": "docs/reference/auth/webauthn.rst", "content": ".. _ref_guide_auth_webauthn:\n\n========\nWebAuthn\n========\n\n:edb-alt-title: Integrating Gel Auth's WebAuthn provider\n\nWebAuthn, short for Web Authentication, is a web standard published by the\nWorld Wide Web Consortium (W3C) for secure and passwordless authentication on\nthe web. It allows users to log in using biometrics, mobile devices, or FIDO2\nsecurity keys instead of traditional passwords. This guide will walk you\nthrough integrating WebAuthn authentication with your application using Gel\nAuth.\n\nWhy choose WebAuthn?\n====================\n\nWebAuthn provides a more secure and user-friendly alternative to passwords and\nSMS-based OTPs. By leveraging public key cryptography, it significantly reduces\nthe risk of phishing, man-in-the-middle, and replay attacks. For application\ndevelopers, integrating WebAuthn can enhance security while improving the user\nexperience with seamless, passwordless logins.\n\nWhat is a Passkey?\n==================\n\nWhile WebAuthn focuses on authenticating users through cryptographic\ncredentials, Passkeys extend this concept by enabling users to easily access\ntheir credentials across devices, including those they haven't used before,\nwithout the need for a password. Passkeys are built on the WebAuthn framework\nand aim to simplify the user experience further by leveraging cloud\nsynchronization of credentials.\n\nMany operating systems and password managers have added support for Passkeys,\nmaking it easier for users to manage their credentials across devices. Gel\nAuth's WebAuthn provider supports Passkeys, allowing users to log in to your\napplication using their Passkeys.\n\nSecurity considerations\n=======================\n\nFor maximum flexibility, Gel Auth's WebAuthn provider allows multiple\nWebAuthn credentials per email. This means that it's very important to verify\nthe email before trusting a WebAuthn credential. This can be done by setting\nthe ``require_verification`` option to ``true`` (which is the default) in your\nWebAuthn provider configuration. Or you can check the verification status of\nthe factor directly.\n\nWebAuthn flow\n=============\n\nThe WebAuthn authentication flow is a sophisticated process that involves a\ncoordinated effort between the server and the client-side script. Unlike the\nother authentication methods outlined elsewhere in this guide, WebAuthn is a\ncoordinated flow that involves a client-side script access web browser APIs, the\nWeb Authentication API specifically, to interact with the user's authenticator\ndevice or passkey.\n\nAt a high level, the sign-up ceremony involves the following steps:\n\n1. The user initiates the sign-up process by providing their email address.\n2. The server generates a JSON object that is used to configure the WebAuthn\n registration ceremony.\n3. The client takes that JSON object, and using the Web Authentication API,\n interacts with the user's authenticator device to create a new credential.\n4. The client sends the credential back to the server.\n5. The server verifies the credential and associates it with the user's email\n address.\n\nThe sign-in ceremony is similar, but instead of creating a new credential, the\nclient uses the Web Authentication API to authenticate the user with an existing\ncredential.\n\nExample implementation\n======================\n\nWe will demonstrate the various steps below by building a NodeJS HTTP server in\na single file that we will use to simulate a typical web application.\n\n.. note::\n\n The details below show the inner workings of how data is exchanged with the\n Auth extension from a web app using HTTP. You can use this as a guide to\n integrate with your application written in any language that can send and\n receive HTTP requests.\n\nStart the PKCE flow\n-------------------\n\nWe secure authentication tokens and other sensitive data by using PKCE\n(Proof Key of Code Exchange).\n\nYour application server creates a 32-byte Base64 URL-encoded string (which will\nbe 43 bytes after encoding), called the ``verifier``. You need to store this\nvalue for the duration of the flow. One way to accomplish this bit of state is\nto use an HttpOnly cookie when the browser makes a request to the server for\nthis value, which you can then use to retrieve it from the cookie store at the\nend of the flow. Take this ``verifier`` string, hash it with SHA256, and then\nbase64url encode the resulting string. This new string is called the\n``challenge``.\n\n.. lint-off\n\n.. code-block:: javascript\n\n import http from \"node:http\";\n import { URL } from \"node:url\";\n import crypto from \"node:crypto\";\n\n /**\n * You can get this value by running `gel instance credentials`.\n * Value should be:\n * `${protocol}://${host}:${port}/branch/${branch}/ext/auth/\n */\n const GEL_AUTH_BASE_URL = process.env.GEL_AUTH_BASE_URL;\n const SERVER_PORT = 3000;\n\n /**\n * Generate a random Base64 url-encoded string, and derive a \"challenge\"\n * string from that string to use as proof that the request for a token\n * later is made from the same user agent that made the original request\n *\n * @returns {Object} The verifier and challenge strings\n */\n const generatePKCE = () => {\n const verifier = crypto.randomBytes(32).toString(\"base64url\");\n\n const challenge = crypto\n .createHash(\"sha256\")\n .update(verifier)\n .digest(\"base64url\");\n\n return { verifier, challenge };\n };\n\n.. lint-on\n\nRouting\n-------\n\nLet's set up the routes we will use to handle the WebAuthn flow. We will then\ndetail each route handler in the following sections.\n\n.. lint-off\n\n.. code-block:: javascript\n\n const server = http.createServer(async (req, res) => {\n const requestUrl = getRequestUrl(req);\n\n switch (requestUrl.pathname) {\n case \"/auth/webauthn/register/options\": {\n await handleRegisterOptions(req, res);\n break;\n }\n\n case \"/auth/webauthn/register\": {\n await handleRegister(req, res);\n break;\n }\n\n case \"/auth/webauthn/authenticate/options\": {\n await handleAuthenticateOptions(req, res);\n break;\n }\n\n case \"/auth/webauthn/authenticate\": {\n await handleAuthenticate(req, res);\n break;\n }\n\n case \"/auth/webauthn/verify\": {\n await handleVerify(req, res);\n break;\n }\n\n default: {\n res.writeHead(404);\n res.end(\"Not found\");\n break;\n }\n }\n });\n\n.. lint-on\n\nHandle register and authenticate options\n----------------------------------------\n\nThe first step in the WebAuthn flow is to get the options for registering a new\ncredential or authenticating an existing credential. The server generates a\nJSON object that is used to configure the WebAuthn registration or\nauthentication ceremony. The Gel Auth extension provides these endpoints\ndirectly, so you can either proxy the request to the Auth extension or redirect\nthe user to the Auth extension's URL. We'll show the proxy option here.\n\n.. lint-off\n\n.. code-block:: javascript\n\n const handleRegisterOptions = async (req, res) => {\n let body = \"\";\n req.on(\"data\", (chunk) => {\n body += chunk.toString();\n });\n req.on(\"end\", async () => {\n const { email } = JSON.parse(body);\n if (!email) {\n res.status = 400;\n res.end(\n `Request body malformed. Expected JSON body with 'email' key, but got: ${body}`,\n );\n return;\n }\n\n const registerUrl = new URL(\n \"webauthn/register/options\",\n GEL_AUTH_BASE_URL\n );\n registerUrl.searchParams.set(\"email\", email);\n\n const registerResponse = await fetch(registerUrl.href);\n\n if (!registerResponse.ok) {\n const text = await registerResponse.text();\n res.status = 400;\n res.end(`Error from the auth server: ${text}`);\n return;\n }\n\n const registerData = await registerResponse.json();\n\n res.writeHead(200, { \"Content-Type\": \"application/json\" });\n res.end(JSON.stringify(registerData));\n });\n };\n\n const handleAuthenticateOptions = async (req, res) => {\n let body = \"\";\n req.on(\"data\", (chunk) => {\n body += chunk.toString();\n });\n req.on(\"end\", async () => {\n const { email } = JSON.parse(body);\n if (!email) {\n res.status = 400;\n res.end(\n `Request body malformed. Expected JSON body with 'email' key, but got: ${body}`,\n );\n return;\n }\n\n const authenticateUrl = new URL(\n \"webauthn/authenticate/options\",\n GEL_AUTH_BASE_URL\n );\n authenticateUrl.searchParams.set(\"email\", email);\n\n const authenticateResponse = await fetch(authenticateUrl.href);\n\n if (!authenticateResponse.ok) {\n const text = await authenticateResponse.text();\n res.status = 400;\n res.end(`Error from the auth server: ${text}`);\n return;\n }\n\n const authenticateData = await authenticateResponse.json();\n\n res.writeHead(200, { \"Content-Type\": \"application/json\" });\n res.end(JSON.stringify(authenticateData));\n });\n };\n\n.. lint-on\n\nRegister a new credential\n-------------------------\n\nThe client script will call the Web Authentication API to create a new\ncredential payload and send it to this endpoint. This endpoints job will be to\nforward the serialized credential payload to the Gel Auth extension for\nverification, and then associate the credential with the user's email address.\n\n.. lint-off\n\n.. code-block:: javascript\n\n const handleRegister = async (req, res) => {\n let body = \"\";\n req.on(\"data\", (chunk) => {\n body += chunk.toString();\n });\n req.on(\"end\", async () => {\n const { challenge, verifier } = generatePKCE();\n const { email, provider, credentials, verify_url, user_handle } = JSON.parse(body);\n if (!email || !provider || !credentials || !verify_url || !user_handle) {\n res.status = 400;\n res.end(\n `Request body malformed. Expected JSON body with 'email', 'provider', 'credentials', 'verify_url', and 'user_handle' keys, but got: ${body}`,\n );\n return;\n }\n\n const registerUrl = new URL(\"webauthn/register\", GEL_AUTH_BASE_URL);\n\n const registerResponse = await fetch(registerUrl.href, {\n method: \"post\",\n headers: {\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({\n provider,\n email,\n credentials,\n verify_url,\n user_handle,\n challenge,\n }),\n });\n\n if (!registerResponse.ok) {\n const text = await registerResponse.text();\n res.status = 400;\n res.end(`Error from the auth server: ${text}`);\n return;\n }\n\n const registerData = await registerResponse.json();\n if (\"code\" in registerData) {\n const tokenUrl = new URL(\"token\", GEL_AUTH_BASE_URL);\n tokenUrl.searchParams.set(\"code\", registerData.code);\n tokenUrl.searchParams.set(\"verifier\", verifier);\n const tokenResponse = await fetch(tokenUrl.href, {\n method: \"get\",\n });\n\n if (!tokenResponse.ok) {\n const text = await authenticateResponse.text();\n res.status = 400;\n res.end(`Error from the auth server: ${text}`);\n return;\n }\n\n const { auth_token } = await tokenResponse.json();\n res.writeHead(204, {\n \"Set-Cookie\": `gel-auth-token=${auth_token}; HttpOnly; Path=/; Secure; SameSite=Strict`,\n });\n res.end();\n } else {\n res.writeHead(204, {\n \"Set-Cookie\": `gel-pkce-verifier=${pkce.verifier}; HttpOnly; Path=/; Secure; SameSite=Strict`,\n });\n res.end();\n }\n });\n };\n\n.. lint-on\n\nAuthenticate with an existing credential\n----------------------------------------\n\nThe client script will call the Web Authentication API to authenticate with an\nexisting credential and send the assertion to this endpoint. This endpoint's\njob will be to forward the serialized assertion to the Gel Auth extension\nfor verification.\n\n.. lint-off\n\n.. code-block:: javascript\n\n const handleAuthenticate = async (req, res) => {\n let body = \"\";\n req.on(\"data\", (chunk) => {\n body += chunk.toString();\n });\n req.on(\"end\", async () => {\n const { challenge, verifier } = generatePKCE();\n const { email, provider, assertion } = JSON.parse(body);\n if (!email || !provider || !assertion) {\n res.status = 400;\n res.end(\n `Request body malformed. Expected JSON body with 'email', 'provider', and 'assertion' keys, but got: ${body}`,\n );\n return;\n }\n\n const authenticateUrl = new URL(\"webauthn/authenticate\", GEL_AUTH_BASE_URL);\n\n const authenticateResponse = await fetch(authenticateUrl.href, {\n method: \"post\",\n headers: {\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({\n provider,\n email,\n assertion,\n challenge,\n }),\n });\n\n if (!authenticateResponse.ok) {\n const text = await authenticateResponse.text();\n res.status = 400;\n res.end(`Error from the auth server: ${text}`);\n return;\n }\n\n const authenticateData = await authenticateResponse.json();\n if (\"code\" in authenticateData) {\n const tokenUrl = new URL(\"token\", GEL_AUTH_BASE_URL);\n tokenUrl.searchParams.set(\"code\", authenticateData.code);\n const tokenResponse = await fetch(tokenUrl.href, {\n method: \"get\",\n });\n\n if (!tokenResponse.ok) {\n const text = await authenticateResponse.text();\n res.status = 400;\n res.end(`Error from the auth server: ${text}`);\n return;\n }\n\n const { auth_token } = await tokenResponse.json();\n res.writeHead(204, {\n \"Set-Cookie\": `gel-auth-token=${auth_token}; HttpOnly; Path=/; Secure; SameSite=Strict`,\n });\n res.end();\n } else {\n res.writeHead(400, { \"Content-Type\": \"application/json\" });\n res.end(JSON.stringify({ error: \"Email must be verified before being able to authenticate.\" }));\n }\n });\n };\n\n.. lint-on\n\nHandle email verification\n-------------------------\n\nWhen a new user signs up, by default we require them to verify their email\naddress before allowing the application to get an authentication token. To\nhandle the verification flow, we implement an endpoint:\n\n.. note::\n\n If your WebAuthn provider uses the **Code** verification method, the\n verification email contains a one-time code rather than a link. In that\n case, prompt the user for the code and call ``POST /verify`` with:\n\n - **provider**: ``builtin::local_webauthn``\n - **email** and **code**\n - optionally a **challenge** and **redirect_to** to receive a PKCE code or a redirect upon success\n\n The Link-based example below continues to work when the provider uses the\n Link method.\n\n.. note::\n\n šŸ’” If you would like to allow users to still log in, but offer limited access\n to your application, you can check the associated\n ``ext::auth::WebAuthnFactor`` for the ``ext::auth::Identity`` to see if the\n ``verified_at`` property is some time in the past. You'll need to set the\n ``require_verification`` setting in the provider configuration to ``false``.\n\n.. lint-off\n\n.. code-block:: javascript\n\n /**\n * Handles the link in the email verification flow.\n *\n * @param {Request} req\n * @param {Response} res\n */\n const handleVerify = async (req, res) => {\n const requestUrl = getRequestUrl(req);\n const verification_token = requestUrl.searchParams.get(\"verification_token\");\n if (!verification_token) {\n res.status = 400;\n res.end(\n `Verify request is missing 'verification_token' search param. The verification email is malformed.`,\n );\n return;\n }\n\n const cookies = req.headers.cookie?.split(\"; \");\n const verifier = cookies\n ?.find((cookie) => cookie.startsWith(\"gel-pkce-verifier=\"))\n ?.split(\"=\")[1];\n if (!verifier) {\n res.status = 400;\n res.end(\n `Could not find 'verifier' in the cookie store. Is this the same user agent/browser that started the authorization flow?`,\n );\n return;\n }\n\n const verifyUrl = new URL(\"verify\", GEL_AUTH_BASE_URL);\n const verifyResponse = await fetch(verifyUrl.href, {\n method: \"post\",\n headers: {\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({\n verification_token,\n verifier,\n provider: \"builtin::local_webauthn\",\n }),\n });\n\n if (!verifyResponse.ok) {\n const text = await verifyResponse.text();\n res.status = 400;\n res.end(`Error from the auth server: ${text}`);\n return;\n }\n\n const { code } = await verifyResponse.json();\n\n const tokenUrl = new URL(\"token\", GEL_AUTH_BASE_URL);\n tokenUrl.searchParams.set(\"code\", code);\n tokenUrl.searchParams.set(\"verifier\", verifier);\n const tokenResponse = await fetch(tokenUrl.href, {\n method: \"get\",\n });\n\n if (!tokenResponse.ok) {\n const text = await tokenResponse.text();\n res.status = 400;\n res.end(`Error from the auth server: ${text}`);\n return;\n }\n\n const { auth_token } = await tokenResponse.json();\n res.writeHead(204, {\n \"Set-Cookie\": `gel-auth-token=${auth_token}; HttpOnly; Path=/; Secure; SameSite=Strict`,\n });\n res.end();\n };\n\n.. lint-on\n\nClient-side script\n------------------\n\nOn the client-side, you will need to write a script that retrieves the options\nfrom the Gel Auth extension, calls the Web Authentication API, and sends the\nresulting credential or assertion to the server. Writing out the low-level\nhandling of serialization and deserialization of the WebAuthn data is beyond the\nscope of this guide, but we publish a WebAuthn client library that you can use\nto simlify this process. The library is available on npm as part of our\n``@gel/auth-core`` library. Here is an example of how you might set up a form\nwith appropriate click handlers to perform the WebAuthn sign in and sign up\nceremonies.\n\n.. lint-off\n\n.. code-block:: javascript\n\n import { WebAuthnClient } from \"@gel/auth-core/webauthn\";\n\n const webAuthnClient = new WebAuthnClient({\n signupOptionsUrl: \"http://localhost:3000/auth/webauthn/register/options\",\n signupUrl: \"http://localhost:3000/auth/webauthn/register\",\n signinOptionsUrl: \"http://localhost:3000/auth/webauthn/authenticate/options\",\n signinUrl: \"http://localhost:3000/auth/webauthn/authenticate\",\n verifyUrl: \"http://localhost:3000/auth/webauthn/verify\",\n });\n\n document.addEventListener(\"DOMContentReady\", () => {\n const signUpButton = document.querySelector(\"button#sign-up\");\n const signInButton = document.querySelector(\"button#sign-in\");\n const emailInput = document.querySelector(\"input#email\");\n\n if (signUpButton) {\n signUpButton.addEventListener(\"click\", async (event) => {\n event.preventDefault();\n const email = emailInput.value.trim();\n if (!email) {\n throw new Error(\"No email provided\");\n }\n try {\n await webAuthnClient.signUp(email);\n window.location = \"http://localhost:3000/signup-success\";\n } catch (err) {\n console.error(err);\n window.location = \"http://localhost:3000/signup-error\";\n }\n });\n }\n\n if (signInButton) {\n signInButton.addEventListener(\"click\", async (event) => {\n event.preventDefault();\n const email = emailInput.value.trim();\n if (!email) {\n throw new Error(\"No email provided\");\n }\n try {\n await webAuthnClient.signIn(email);\n window.location = \"http://localhost:3000\";\n } catch (err) {\n console.error(err);\n window.location = \"http://localhost:3000/signup-error\";\n }\n })\n }\n });\n\n.. lint-on\n" }, { "path": "docs/reference/auth/webhooks.rst", "content": ".. _ref_auth_webhooks:\n\n========\nWebhooks\n========\n\nThe auth extension supports sending webhooks for a variety of auth events. You can use these webhooks to, for instance, send a fully customized email for email verification, or password reset instead of our built-in email verification and password reset emails. You could also use them to trigger analytics events, start an email drip campaign, create an audit log, or trigger other side effects in your application.\n\nIf you are using Webhooks to send emails, be sure to not also configure an SMTP provider otherwise we will send the email via SMTP and also send the webhook which will trigger your custom email sending behavior.\n\n.. warning::\n\n We send webhooks with no durability or reliability guarantees, so you should always provide a mechanism for retrying delivery of any critical events, such as email verification and password reset. We detail how to resend these events in the relevant sections on the various authentication flows.\n\nConfiguration\n=============\n\nYou can configure webhooks with the UI or via query. The URLs you register as webhooks must be unique across all webhooks configured for each branch. If you want to send multiple events to the same URL, you can do so by adding multiple ``ext::auth::WebhookEvent`` values to the ``events`` set, like in this example.\n\n.. code-block:: edgeql\n\n configure current branch insert\n ext::auth::WebhookConfig {\n url := 'https://example.com/auth/webhook',\n events := {\n ext::auth::WebhookEvent.EmailVerificationRequested,\n ext::auth::WebhookEvent.PasswordResetRequested,\n },\n # Optional, only needed if you want to verify the webhook request\n signing_secret_key := '1234567890',\n };\n\nWhen you receive a webhook, you'll look at the ``event_type`` field to determine which event corresponds to this webhook request and handle it accordingly.\n\nChecking webhook signatures\n===========================\n\nYou can provide a signing key, which you will need to generate and save in a place that your application will have access to. The extension will then add a ``x-ext-auth-signature-sha256`` header to the request, which you can use to verify the request by comparing the signature to the SHA256 hash of the request body.\n\nHere is an example of how you might verify the signature in a Node.js application:\n\n.. code-block:: typescript\n\n /**\n * Assert that if the request contains a signature header, that the signature\n * is valid for the request body. Will return false if there is no signature\n * header.\n *\n * @param {Request} request - The request to verify.\n * @param {string} signingKey - The key to use to verify the signature.\n * @returns {boolean} - True if the signature is present and valid, false if\n * the signature is not present at all.\n * @throws {AssertionError} - If the signature is present but invalid.\n */\n async function assertSignature(\n request: Request,\n signingKey: string,\n ): Promise {\n const signatureHeader = request.headers.get('x-ext-auth-signature-sha256');\n if (!signatureHeader) {\n return false;\n }\n\n const requestBody = await request.text();\n const encoder = new TextEncoder();\n const data = encoder.encode(requestBody);\n const key = await crypto.subtle.importKey(\n 'raw',\n encoder.encode(signingKey),\n { name: 'HMAC', hash: 'SHA-256' },\n false,\n ['sign']\n );\n const signature = await crypto.subtle.sign('HMAC', key, data);\n const signatureHex = Buffer.from(signature).toString('hex');\n\n assert.strictEqual(\n signatureHeader,\n signatureHex,\n \"Signature header is set, but the signature is invalid\"\n );\n\n return true;\n };\n\n\nTroubleshooting webhooks\n========================\n\nIf you are having trouble receiving webhooks, you might need to look for any responses from the requests that are being scheduled by the :ref:`std::net::http ` module. You can list all of the :eql:type:`net::http::ScheduledRequest` objects, and any returned responses with the following query:\n\n.. code-block:: edgeql\n\n select net::http::ScheduledRequest {\n **,\n response: { ** }\n }\n\nEvents reference\n================\n\nCommon fields for all events:\n\n* ``event_type``: (string) This will be a literal string containing the name of the event. You can use this to determine which event occurred.\n* ``event_id``: (string) A unique identifier to help disambiguate events of the same type.\n* ``timestamp``: (string) The ISO 8601 timestamp of when the event was triggered.\n\n\nIdentity created\n^^^^^^^^^^^^^^^^\n\nWhen a new ``ext::auth::Identity`` object is created, like when a new user signs up, or an existing user adds a new factor, this event is triggered.\n\n**Example payload:**\n\n.. code-block:: text\n\n POST http://localhost:8000/auth/webhook\n Content-type: application/json\n x-ext-auth-signature-sha256: 1234567890\n\n {\n \"event_type\": \"IdentityCreated\",\n \"event_id\": \"1234567890\",\n \"timestamp\": \"2021-01-01T00:00:00Z\",\n \"identity_id\": \"identity123\"\n }\n\nIdentity authenticated\n^^^^^^^^^^^^^^^^^^^^^^\n\nWhen an ``ext::auth::Identity`` object is authenticated, like when a user logs in, this event is triggered.\n\n**Example payload:**\n\n.. code-block:: text\n\n POST http://localhost:8000/auth/webhook\n Content-type: application/json\n x-ext-auth-signature-sha256: 1234567890\n\n {\n \"event_type\": \"IdentityAuthenticated\",\n \"event_id\": \"1234567890\",\n \"timestamp\": \"2021-01-01T00:00:00Z\",\n \"identity_id\": \"identity123\"\n }\n\nEmail factor created\n^^^^^^^^^^^^^^^^^^^^\n\nWhen a new ``ext::auth::EmailFactor`` object is created, like when a user adds a new email factor, this event is triggered.\n\n**Example payload:**\n\n.. code-block:: text\n\n POST http://localhost:8000/auth/webhook\n Content-type: application/json\n x-ext-auth-signature-sha256: 1234567890\n\n {\n \"event_type\": \"EmailFactorCreated\",\n \"event_id\": \"1234567890\",\n \"timestamp\": \"2021-01-01T00:00:00Z\",\n \"identity_id\": \"identity123\",\n \"email_factor_id\": \"emailfactor123\"\n }\n\nEmail verified\n^^^^^^^^^^^^^^\n\nWhen a user verifies their email address, this event is triggered.\n\n**Example payload:**\n\n.. code-block:: text\n\n POST http://localhost:8000/auth/webhook\n Content-type: application/json\n x-ext-auth-signature-sha256: 1234567890\n\n {\n \"event_type\": \"EmailVerified\",\n \"event_id\": \"1234567890\",\n \"timestamp\": \"2021-01-01T00:00:00Z\",\n \"identity_id\": \"identity123\",\n \"email_factor_id\": \"emailfactor123\"\n }\n\nEmail verification requested\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nWhen a user requests to verify their email address, like when they first sign up, or requests to resend the verification email, this event is triggered.\n\n**Example payload:**\n\n.. code-block:: text\n\n POST http://localhost:8000/auth/webhook\n Content-type: application/json\n x-ext-auth-signature-sha256: 1234567890\n\n {\n \"event_type\": \"EmailVerificationRequested\",\n \"event_id\": \"1234567890\",\n \"timestamp\": \"2021-01-01T00:00:00Z\",\n \"identity_id\": \"identity123\",\n \"verification_token\": \"verificationtoken123\"\n }\n\nPassword reset requested\n^^^^^^^^^^^^^^^^^^^^^^^^\n\nWhen a user requests to reset their password, this event is triggered.\n\n**Example payload:**\n\n.. code-block:: text\n\n POST http://localhost:8000/auth/webhook\n Content-type: application/json\n x-ext-auth-signature-sha256: 1234567890\n\n {\n \"event_type\": \"PasswordResetRequested\",\n \"event_id\": \"1234567890\",\n \"timestamp\": \"2021-01-01T00:00:00Z\",\n \"identity_id\": \"identity123\",\n \"reset_token\": \"resettoken123\"\n }\n\nMagic link requested\n^^^^^^^^^^^^^^^^^^^^\n\nWhen a user requests to send a magic link email, like for signing in, or signing up for the first time, this event is triggered.\n\n**Example payload:**\n\n.. code-block:: text\n\n POST http://localhost:8000/auth/webhook\n Content-type: application/json\n x-ext-auth-signature-sha256: 1234567890\n\n {\n \"event_type\": \"MagicLinkRequested\",\n \"event_id\": \"1234567890\",\n \"timestamp\": \"2021-01-01T00:00:00Z\",\n \"identity_id\": \"identity123\",\n \"email_factor_id\": \"emailfactor123\",\n \"magic_link_token\": \"magiclinktoken123\",\n \"magic_link_url\": \"http://localhost:8000/auth/magic-link?token=magiclinktoken123\"\n }\n" }, { "path": "docs/reference/datamodel/access_policies.rst", "content": ".. _ref_datamodel_access_policies:\n\n===============\nAccess Policies\n===============\n\n.. index:: object-level security, row-level security, RLS\n\nObject types in |Gel| can contain security policies that restrict the set of\nobjects that can be selected, inserted, updated, or deleted by a particular\nquery. This is known as *object-level security* and is similar in function\nto SQL's row-level security.\n\nWhen no access policies are defined, object-level security is not activated:\nany properly authenticated client can carry out any operation on any object\nin the database. Access policies allow you to ensure that the database itself\nhandles access control logic rather than having to implement it in every\napplication or service that connects to your database.\n\nAccess policies can greatly simplify your backend code, centralizing access\ncontrol logic in a single place. They can also be extremely useful for\nimplementing AI agentic flows, where you want to have guardrails around\nyour data that agents can't break.\n\nWe'll illustrate access policies in this document with this simple schema:\n\n.. code-block:: sdl\n\n type User {\n required email: str { constraint exclusive; }\n }\n\n type BlogPost {\n required title: str;\n required author: User;\n }\n\n\n.. warning::\n\n Once a policy is added to a particular object type, **all operations**\n (``select``, ``insert``, ``delete``, ``update``, etc.) on any object of\n that type are now *disallowed by default* unless specifically allowed by\n an access policy! See :ref:`resolution order `\n below for details.\n\nGlobal variables\n================\n\nGlobal variables are a convenient way to set up the context for your access\npolicies. Gel's global variables are tightly integrated with the Gel's\ndata model, client APIs, EdgeQL and SQL, and the tooling around them.\n\nGlobal variables in Gel are not pre-defined. Users are free to define\nas many globals in their schema as they want to represent the business\nlogic of their application.\n\nA common scenario is storing a ``current_user`` global representing\nthe user executing queries. We'd like to have a slightly more complex example\nshowing that you can use more than one global variable. Let's do that:\n\n* We'll use one *global* ``uuid`` to represent the identity of the user\n executing the query.\n* We'll have the ``Country`` *enum* to represent the type of country\n that the user is currently in. The enum represents three types of\n countries: those where the service has not been rolled out, those with\n read-only access, and those with full access.\n* We'll use the ``current_country`` *global* to represent the user's\n current country. In our *example schema*, we want *country* to be\n context-specific: the same user who can access certain content in one\n country might not be able to in another country (let's imagine that's\n due to different country-specific legal frameworks).\n\nHere is an illustration:\n\n.. code-block:: sdl-diff\n\n + scalar type Country extending enum;\n + global current_user: uuid;\n + required global current_country: Country {\n + default := Country.None\n + }\n\n type User {\n required email: str { constraint exclusive; }\n }\n\n type BlogPost {\n required title: str;\n required author: User;\n }\n\nYou can set and reset these globals in Gel client libraries, for example:\n\n.. tabs::\n\n .. code-tab:: typescript\n\n import createClient from 'gel';\n\n const client = createClient();\n\n // 'authedClient' will share the network connection with 'client',\n // but will have the 'current_user' global set.\n const authedClient = client.withGlobals({\n current_user: '2141a5b4-5634-4ccc-b835-437863534c51',\n });\n\n const result = await authedClient.query(\n `select global current_user;`);\n console.log(result);\n\n .. code-tab:: python\n\n from gel import create_client\n\n client = create_client().with_globals({\n 'current_user': '580cc652-8ab8-4a20-8db9-4c79a4b1fd81'\n })\n\n result = client.query(\"\"\"\n select global current_user;\n \"\"\")\n print(result)\n\n .. code-tab:: go\n\n package main\n\n import (\n \"context\"\n \"fmt\"\n \"log\"\n\n \"github.com/geldata/gel-go\"\n )\n\n func main() {\n ctx := context.Background()\n client, err := gel.CreateClient(ctx, gel.Options{})\n if err != nil {\n log.Fatal(err)\n }\n defer client.Close()\n\n id, err := gel.ParseUUID(\"2141a5b4-5634-4ccc-b835-437863534c51\")\n if err != nil {\n log.Fatal(err)\n }\n\n var result gel.UUID\n err = client.\n WithGlobals(map[string]interface{}{\"current_user\": id}).\n QuerySingle(ctx, \"SELECT global current_user;\", &result)\n if err != nil {\n log.Fatal(err)\n }\n\n fmt.Println(result)\n }\n\n .. code-tab:: rust\n\n use gel_protocol::{\n model::Uuid,\n value::EnumValue\n };\n\n let client = gel_tokio::create_client()\n .await\n .expect(\"Client should init\")\n .with_globals_fn(|c| {\n c.set(\n \"current_user\",\n Value::Uuid(\n Uuid::parse_str(\"2141a5b4-5634-4ccc-b835-437863534c51\")\n .expect(\"Uuid should have parsed\"),\n ),\n );\n c.set(\n \"current_country\",\n Value::Enum(EnumValue::from(\"Full\"))\n );\n });\n client\n .query_required_single::(\"select global current_user;\", &())\n .await\n .expect(\"Returning value\");\n\n\nDefining policies\n=================\n\nA policy example for our simple blog schema might look like:\n\n.. code-block:: sdl-diff\n\n global current_user: uuid;\n required global current_country: Country {\n default := Country.None\n }\n scalar type Country extending enum;\n\n type User {\n required email: str { constraint exclusive; }\n }\n\n type BlogPost {\n required title: str;\n required author: User;\n\n + access policy author_has_full_access\n + allow all\n + using (global current_user ?= .author.id\n + and global current_country ?= Country.Full) {\n + errmessage := \"User does not have full access\";\n + }\n\n + access policy author_has_read_access\n + allow select\n + using (global current_user ?= .author.id\n + and global current_country ?= Country.ReadOnly);\n }\n\nExplanation:\n\n- ``access policy `` introduces a new policy in an object type.\n- ``allow all`` grants ``select``, ``insert``, ``update``, and ``delete``\n access if the condition passes. We also used a separate policy to allow\n only ``select`` in some cases.\n- ``using ()`` is a boolean filter restricting the set of objects to\n which the policy applies. (We used the coalescing operator ``?=`` to\n handle empty sets gracefully.)\n- ``errmessage`` is an optional custom message to display in case of a write\n violation.\n\nLet's run some experiments in the REPL:\n\n.. code-block:: edgeql-repl\n\n db> insert User { email := \"test@example.com\" };\n {default::User {id: be44b326-03db-11ed-b346-7f1594474966}}\n db> set global current_user :=\n ... \"be44b326-03db-11ed-b346-7f1594474966\";\n OK: SET GLOBAL\n db> set global current_country := Country.Full;\n OK: SET GLOBAL\n db> insert BlogPost {\n ... title := \"My post\",\n ... author := (select User filter .id = global current_user)\n ... };\n {default::BlogPost {id: e76afeae-03db-11ed-b346-fbb81f537ca6}}\n\nBecause the user is in a \"full access\" country and the current user ID\nmatches the author, the new blog post is permitted. When the same user sets\n``global current_country := Country.ReadOnly;``:\n\n.. code-block:: edgeql-repl\n\n db> set global current_country := Country.ReadOnly;\n OK: SET GLOBAL\n db> select BlogPost;\n {default::BlogPost {id: e76afeae-03db-11ed-b346-fbb81f537ca6}}\n db> insert BlogPost {\n ... title := \"My second post\",\n ... author := (select User filter .id = global current_user)\n ... };\n gel error: AccessPolicyError: access policy violation on\n insert of default::BlogPost (User does not have full access)\n\nFinally, let's unset ``current_user`` and see how many blog posts are returned\nwhen we count them.\n\n.. code-block:: edgeql-repl\n\n db> set global current_user := {};\n OK: SET GLOBAL\n db> select BlogPost;\n {}\n db> select count(BlogPost);\n {0}\n\n``select BlogPost`` returns zero results in this case as well. We can only\n``select`` the *posts* written by the *user* specified by ``current_user``.\nWhen ``current_user`` has no value or has a different value from the\n``.author.id`` of any existing ``BlogPost`` objects, we can't read any posts.\nBut thanks to ``Country`` being set to ``Country.Full``, this user will be\nable to write a new blog post.\n\n**The bottom line:** access policies use global variables to define a\n\"subgraph\" of data that is visible to your queries.\n\n\nPolicy types\n============\n\n.. api-index:: select, insert, delete, update read, update write, all\n\nThe types of policy rules map to the statement type in EdgeQL:\n\n- ``select``: Controls which objects are visible to any query.\n- ``insert``: Post-insert check. If the inserted object violates the policy,\n the operation fails.\n- ``delete``: Controls which objects can be deleted.\n- ``update read``: Pre-update check on which objects can be updated at all.\n- ``update write``: Post-update check for how objects can be updated.\n- ``all``: Shorthand for granting or denying ``select, insert, update,\n delete``.\n\nResolution order\n================\n\nIf multiple policies apply (some are ``allow`` and some are ``deny``), the\nlogic is:\n\n1. If there are no policies, access is allowed.\n2. All ``allow`` policies collectively form a *union* / *or* of allowed sets.\n3. All ``deny`` policies *subtract* from that union, overriding allows!\n4. The final set of objects is the intersection of the above logic for each\n operation: ``select, insert, update read, update write, delete``.\n\nBy default, once you define any policy on an object type, you must explicitly\nallow the operations you need. This is a common **pitfall** when you are\nstarting out with access policies (but you will develop an intuition for this\nquickly). Let's look at an example:\n\n.. code-block:: sdl\n\n global current_user_id: uuid;\n global current_user := (\n select User filter .id = global current_user_id\n );\n\n type User {\n required email: str { constraint exclusive; }\n required is_admin: bool { default := false };\n\n access policy admin_only\n allow all\n using (global current_user.is_admin ?? false);\n }\n\n type BlogPost {\n required title: str;\n author: User;\n\n access policy author_has_full_access\n allow all\n using (global current_user ?= .author.id);\n }\n\nIn the above schema only admins will see a non-empty ``author`` link when\nrunning ``select BlogPost { author }``. Why? Because only admins can see\n``User`` objects at all: ``admin_only`` policy is the only one defined on\nthe ``User`` type!\n\nThis means that instead of making ``BlogPost`` visible to its author, all\nnon-admin authors won't be able to see their own posts. The above issue can be\nremedied by making the current user able to see their own ``User`` record.\n\n\nInteraction between policies\n============================\n\nPolicy expressions themselves do not take other policies into account\n(since |EdgeDB| 3). This makes it easier to reason about policies.\n\nCustom error messages\n=====================\n\nWhen an ``insert`` or ``update write`` violates an access policy, Gel will\nraise a generic ``AccessPolicyError``:\n\n.. code-block::\n\n gel error: AccessPolicyError: access policy violation\n on insert of \n\n.. note::\n\n Restricted access is represented either as an error message or an empty\n set, depending on the filtering order of the operation. The operations\n ``select``, ``delete``, or ``update read`` filter up front, and thus you\n simply won't get the data that is being restricted. Other operations\n (``insert`` and ``update write``) will return an error message.\n\nIf multiple policies are in effect, it can be helpful to define a distinct\n``errmessage`` in your policy:\n\n.. code-block:: sdl-diff\n\n global current_user_id: uuid;\n global current_user := (\n select User filter .id = global current_user_id\n );\n\n type User {\n required email: str { constraint exclusive; };\n required is_admin: bool { default := false };\n\n access policy admin_only\n allow all\n + using (global current_user.is_admin ?? false) {\n + errmessage := 'Only admins may query Users'\n + };\n }\n\n type BlogPost {\n required title: str;\n author: User;\n\n access policy author_has_full_access\n allow all\n + using (global current_user ?= .author) {\n + errmessage := 'BlogPosts may only be queried by their authors'\n + };\n }\n\nNow if you attempt, for example, a ``User`` insert as a non-admin user, you\nwill receive this error:\n\n.. code-block::\n\n gel error: AccessPolicyError: access policy violation on insert of\n default::User (Only admins may query Users)\n\n\nDisabling policies\n==================\n\n.. api-index:: apply_access_policies\n\nYou may disable all access policies by setting the ``apply_access_policies``\n:ref:`configuration parameter ` to ``false``.\n\nYou may also temporarily disable access policies using the Gel UI configuration\ncheckbox (or via :gelcmd:`ui`), which only applies to your UI session.\n\nMore examples\n=============\n\nHere are some additional patterns:\n\n1. Publicly visible blog posts, only writable by the author:\n\n .. code-block:: sdl-diff\n\n global current_user: uuid;\n\n type User {\n required email: str { constraint exclusive; }\n }\n\n type BlogPost {\n required title: str;\n required author: User;\n + required published: bool { default := false };\n\n access policy author_has_full_access\n allow all\n using (global current_user ?= .author.id);\n + access policy visible_if_published\n + allow select\n + using (.published);\n }\n\n2. Visible to friends, only modifiable by the author:\n\n .. code-block:: sdl-diff\n\n global current_user: uuid;\n\n type User {\n required email: str { constraint exclusive; }\n + multi friends: User;\n }\n\n type BlogPost {\n required title: str;\n required author: User;\n\n access policy author_has_full_access\n allow all\n using (global current_user ?= .author.id);\n + access policy friends_can_read\n + allow select\n + using ((global current_user in .author.friends.id) ?? false);\n }\n\n3. Publicly visible except to those blocked by the author:\n\n .. code-block:: sdl-diff\n\n type User {\n required email: str { constraint exclusive; }\n + multi blocked: User;\n }\n\n type BlogPost {\n required title: str;\n required author: User;\n\n access policy author_has_full_access\n allow all\n using (global current_user ?= .author.id);\n + access policy anyone_can_read\n + allow select;\n + access policy exclude_blocked\n + deny select\n + using ((global current_user in .author.blocked.id) ?? false);\n }\n\n4. \"Disappearing\" posts that become invisible after 24 hours:\n\n .. code-block:: sdl-diff\n\n type User {\n required email: str { constraint exclusive; }\n }\n\n type BlogPost {\n required title: str;\n required author: User;\n + required created_at: datetime {\n + default := datetime_of_statement() # non-volatile\n + }\n\n access policy author_has_full_access\n allow all\n using (global current_user ?= .author.id);\n + access policy hide_after_24hrs\n + allow select\n + using (\n + datetime_of_statement() - .created_at < '24 hours'\n + );\n }\n\nSuper constraints\n=================\n\nAccess policies can act like \"super constraints.\" For instance, a policy on\n``insert`` or ``update write`` can do a post-write validity check, rejecting\nthe operation if a certain condition is not met.\n\nE.g. here's a policy that limits the number of blog posts a\n``User`` can post:\n\n.. code-block:: sdl-diff\n\n type User {\n required email: str { constraint exclusive; }\n + multi posts := . 500);\n }\n\n.. _ref_eql_sdl_access_policies:\n.. _ref_eql_sdl_access_policies_syntax:\n\nDeclaring access policies\n=========================\n\n.. api-index:: access policy, when, allow, deny, all, select, insert, delete,\n update, update read, update write, using, errmessage\n\nThis section describes the syntax to declare access policies in your schema.\n\nSyntax\n------\n\n.. sdl:synopsis::\n\n access policy \n [ when () ]\n { allow | deny } [, ... ]\n [ using () ]\n [ \"{\"\n [ errmessage := value ; ]\n [ ]\n \"}\" ] ;\n\n # where is one of\n all\n select\n insert\n delete\n update [{ read | write }]\n\nWhere:\n\n:eql:synopsis:``\n The name of the access policy.\n\n:eql:synopsis:`when ()`\n Specifies which objects this policy applies to. The\n :eql:synopsis:`` has to be a :eql:type:`bool` expression.\n\n When omitted, it is assumed that this policy applies to all objects of a\n given type.\n\n:eql:synopsis:`allow`\n Indicates that qualifying objects should allow access under this policy.\n\n:eql:synopsis:`deny`\n Indicates that qualifying objects should *not* allow access under this\n policy. This flavor supersedes any :eql:synopsis:`allow` policy and can\n be used to selectively deny access to a subset of objects that otherwise\n explicitly allows accessing them.\n\n:eql:synopsis:`all`\n Apply the policy to all actions. It is exactly equivalent to listing\n :eql:synopsis:`select`, :eql:synopsis:`insert`, :eql:synopsis:`delete`,\n :eql:synopsis:`update` actions explicitly.\n\n:eql:synopsis:`select`\n Apply the policy to all selection queries. Note that any object that\n cannot be selected, cannot be modified either. This makes\n :eql:synopsis:`select` the most basic \"visibility\" policy.\n\n:eql:synopsis:`insert`\n Apply the policy to all inserted objects. If a newly inserted object would\n violate this policy, an error is produced instead.\n\n:eql:synopsis:`delete`\n Apply the policy to all objects about to be deleted. If an object does not\n allow access under this kind of policy, it is not going to be considered\n by any :eql:stmt:`delete` command.\n\n Note that any object that cannot be selected, cannot be modified either.\n\n:eql:synopsis:`update read`\n Apply the policy to all objects selected for an update. If an object does\n not allow access under this kind of policy, it is not visible cannot be\n updated.\n\n Note that any object that cannot be selected, cannot be modified either.\n\n:eql:synopsis:`update write`\n Apply the policy to all objects at the end of an update. If an updated\n object violates this policy, an error is produced instead.\n\n Note that any object that cannot be selected, cannot be modified either.\n\n:eql:synopsis:`update`\n This is just a shorthand for :eql:synopsis:`update read` and\n :eql:synopsis:`update write`.\n\n Note that any object that cannot be selected, cannot be modified either.\n\n:eql:synopsis:`using `\n Specifies what the policy is with respect to a given eligible (based on\n :eql:synopsis:`when` clause) object. The :eql:synopsis:`` has to be\n a :eql:type:`bool` expression. The specific meaning of this value also\n depends on whether this policy flavor is :eql:synopsis:`allow` or\n :eql:synopsis:`deny`.\n\n The expression must be :ref:`Stable `.\n\n When omitted, it is assumed that this policy applies to all eligible\n objects of a given type.\n\n:eql:synopsis:`set errmessage := `\n Set a custom error message of :eql:synopsis:`` that is displayed\n when this access policy prevents a write action.\n\n:sdl:synopsis:``\n Set access policy :ref:`annotation `\n to a given *value*.\n\nAny sub-type extending a type inherits all of its access policies.\nYou can define additional access policies on sub-types.\n\n\n.. _ref_eql_ddl_access_policies:\n\nDDL commands\n============\n\nThis section describes the low-level DDL commands for creating, altering, and\ndropping access policies. You typically don't need to use these commands\ndirectly, but knowing about them is useful for reviewing migrations.\n\nCreate access policy\n--------------------\n\n:eql-statement:\n\nDefine a new object access policy on a type:\n\n.. eql:synopsis::\n\n [ with [, ...] ]\n { create | alter } type \"{\"\n [ ... ]\n create access policy \n [ when () ; ]\n { allow | deny } action [, action ... ; ]\n [ using () ; ]\n [ \"{\"\n [ set errmessage := value ; ]\n [ create annotation := value ; ]\n \"}\" ]\n \"}\"\n\n # where is one of\n all\n select\n insert\n delete\n update [{ read | write }]\n\nSee the meaning of each parameter in the `Declaring access policies`_ section.\n\nThe following subcommands are allowed in the ``create access policy`` block:\n\n:eql:synopsis:`set errmessage := `\n Set a custom error message of :eql:synopsis:`` that is displayed\n when this access policy prevents a write action.\n\n:eql:synopsis:`create annotation := `\n Set access policy annotation :eql:synopsis:`` to\n :eql:synopsis:``.\n\n See :eql:stmt:`create annotation` for details.\n\n\nAlter access policy\n-------------------\n\n:eql-statement:\n\nModify an existing access policy:\n\n.. eql:synopsis::\n\n [ with [, ...] ]\n alter type \"{\"\n [ ... ]\n alter access policy \"{\"\n [ when () ; ]\n [ reset when ; ]\n { allow | deny } [, ... ; ]\n [ using () ; ]\n [ set errmessage := value ; ]\n [ reset expression ; ]\n [ create annotation := ; ]\n [ alter annotation := ; ]\n [ drop annotation ; ]\n \"}\"\n \"}\"\n\nYou can change the policy's condition, actions, or error message, or add/drop\nannotations.\n\nThe parameters describing the action policy are identical to the parameters\nused by ``create action policy``. There are a handful of additional\nsubcommands that are allowed in the ``alter access policy`` block:\n\n:eql:synopsis:`reset when`\n Clear the :eql:synopsis:`when ()` so that the policy applies to\n all objects of a given type. This is equivalent to ``when (true)``.\n\n:eql:synopsis:`reset expression`\n Clear the :eql:synopsis:`using ()` so that the policy always\n passes. This is equivalent to ``using (true)``.\n\n:eql:synopsis:`alter annotation ;`\n Alter access policy annotation :eql:synopsis:``.\n See :eql:stmt:`alter annotation` for details.\n\n:eql:synopsis:`drop annotation ;`\n Remove access policy annotation :eql:synopsis:``.\n See :eql:stmt:`drop annotation` for details.\n\n\nAll the subcommands allowed in the ``create access policy`` block are also\nvalid subcommands for ``alter access policy`` block.\n\nDrop access policy\n------------------\n\n:eql-statement:\n\nRemove an existing policy:\n\n.. eql:synopsis::\n\n [ with [, ...] ]\n alter type \"{\"\n [ ... ]\n drop access policy ;\n \"}\"\n" }, { "path": "docs/reference/datamodel/aliases.rst", "content": ".. _ref_datamodel_aliases:\n\n=======\nAliases\n=======\n\n.. index:: alias, virtual type\n\nYou can think of *aliases* as a way to give schema names to arbitrary EdgeQL\nexpressions. You can later refer to aliases in queries and in other aliases.\n\nAliases are functionally equivalent to expression aliases defined in EdgeQL\nstatements in :ref:`with block `, but are available\nto all queries using the schema and can be introspected.\n\nLike computed properties, the aliased expression is evaluated on the fly\nwhenever the alias is referenced.\n\n\nScalar alias\n============\n\n.. code-block:: sdl\n\n # in your schema:\n alias digits := {0,1,2,3,4,5,6,7,8,9};\n\nLater, in some query:\n\n.. code-block:: edgeql\n\n select count(digits);\n\n\nObject type alias\n=================\n\nThe name of a given object type (e.g. ``User``) is itself a pointer to the *set\nof all User objects*. After declaring the alias below, you can use ``User`` and\n``UserAlias`` interchangeably:\n\n.. code-block:: sdl\n\n alias UserAlias := User;\n\nObject type alias with computeds\n================================\n\nObject type aliases can include a *shape* that declares additional computed\nproperties or links:\n\n.. code-block:: sdl\n\n type Post {\n required title: str;\n }\n\n alias PostWithTrimmedTitle := Post {\n trimmed_title := str_trim(.title)\n }\n\nLater, in some query:\n\n.. code-block:: edgeql\n\n select PostWithTrimmedTitle {\n trimmed_title\n };\n\nArbitrary expressions\n=====================\n\nAliases can correspond to any arbitrary EdgeQL expression, including entire\nqueries.\n\n.. code-block:: sdl\n\n # Tuple alias\n alias Color := (\"Purple\", 128, 0, 128);\n\n # Named tuple alias\n alias GameInfo := (\n name := \"Li Europan Lingues\",\n country := \"Iceland\",\n date_published := 2023,\n creators := (\n (name := \"Bob Bobson\", age := 20),\n (name := \"Trina TrinadĆ³ttir\", age := 25),\n ),\n );\n\n type BlogPost {\n required title: str;\n required is_published: bool;\n }\n\n # Query alias\n alias PublishedPosts := (\n select BlogPost\n filter .is_published = true\n );\n\n.. note::\n\n All aliases are reflected in the database's built-in :ref:`GraphQL schema\n `.\n\n\n.. _ref_eql_sdl_aliases:\n.. _ref_eql_sdl_aliases_syntax:\n\nDefining aliases\n================\n\n.. api-index:: alias\n\nSyntax\n------\n\nDefine a new alias corresponding to the :ref:`more explicit DDL\ncommands `.\n\n.. sdl:synopsis::\n\n alias := ;\n\n alias \"{\"\n using ;\n [ ]\n \"}\" ;\n\nWhere:\n\n:eql:synopsis:``\n The name (optionally module-qualified) of an alias to be created.\n\n:eql:synopsis:``\n The aliased expression. Must be a :ref:`Stable `\n EdgeQL expression.\n\nThe valid SDL sub-declarations are listed below:\n\n:sdl:synopsis:``\n Set alias :ref:`annotation `\n to a given *value*.\n\n\n.. _ref_eql_ddl_aliases:\n\nDDL commands\n============\n\nThis section describes the low-level DDL commands for creating and\ndropping aliases. You typically don't need to use these commands\ndirectly, but knowing about them is useful for reviewing migrations.\n\nCreate alias\n------------\n\n:eql-statement:\n:eql-haswith:\n\nDefine a new alias in the schema.\n\n.. eql:synopsis::\n\n [ with [, ...] ]\n create alias := ;\n\n [ with [, ...] ]\n create alias \"{\"\n using ;\n [ create annotation := ; ... ]\n \"}\" ;\n\n # where is:\n\n [ := ] module \n\nParameters\n^^^^^^^^^^\n\nMost sub-commands and options of this command are identical to the\n:ref:`SDL alias declaration `, with some\nadditional features listed below:\n\n:eql:synopsis:`[ := ] module `\n An optional list of module alias declarations to be used in the\n alias definition.\n\n:eql:synopsis:`create annotation := ;`\n An optional list of annotation values for the alias.\n See :eql:stmt:`create annotation` for details.\n\nExample\n^^^^^^^\n\nCreate a new alias:\n\n.. code-block:: edgeql\n\n create alias Superusers := (\n select User filter User.groups.name = 'Superusers'\n );\n\n\nDrop alias\n----------\n\n:eql-statement:\n:eql-haswith:\n\nRemove an alias from the schema.\n\n.. eql:synopsis::\n\n [ with [, ...] ]\n drop alias ;\n\nParameters\n^^^^^^^^^^\n\n*alias-name*\n The name (optionally qualified with a module name) of an existing\n expression alias.\n\nExample\n^^^^^^^\n\nRemove an alias:\n\n.. code-block:: edgeql\n\n drop alias SuperUsers;\n" }, { "path": "docs/reference/datamodel/annotations.rst", "content": ".. _ref_datamodel_annotations:\n.. _ref_eql_sdl_annotations:\n\n===========\nAnnotations\n===========\n\n*Annotations* are named values associated with schema items and are\ndesigned to hold arbitrary schema-level metadata represented as a\n:eql:type:`str` (unstructured text).\n\nUsers can store JSON-encoded data in annotations if they need to store\nmore complex metadata.\n\n\nStandard annotations\n====================\n\n.. api-index:: title, description, deprecated\n\nThere are a number of annotations defined in the standard library. The\nfollowing are the annotations which can be set on any schema item:\n\n- ``std::title``\n- ``std::description``\n- ``std::deprecated``\n\nFor example, consider the following declaration:\n\n.. code-block:: sdl\n\n type Status {\n annotation title := 'Activity status';\n annotation description := 'All possible user activities';\n\n required name: str {\n constraint exclusive\n }\n }\n\nAnd the ``std::deprecated`` annotation can be used to mark deprecated items\n(e.g., :eql:func:`str_rpad`) and to provide some information such as what\nshould be used instead.\n\nUser-defined annotations\n========================\n\nTo declare a custom annotation type beyond the three built-ins, add an abstract\nannotation type to your schema. A custom annotation could be used to attach\narbitrary JSON-encoded data to your schemaā€”potentially useful for introspection\nand code generation.\n\n.. code-block:: sdl\n\n abstract annotation admin_note;\n\n type Status {\n annotation admin_note := 'system-critical';\n }\n\n\n.. _ref_eql_sdl_annotations_syntax:\n\nDeclaring annotations\n=====================\n\n.. api-index:: abstract, inheritable, annotation\n\nThis section describes the syntax to use annotations in your schema.\n\nSyntax\n------\n\n.. sdl:synopsis::\n\n # Abstract annotation form:\n abstract [ inheritable ] annotation \n [ \"{\" ; [...] \"}\" ] ;\n\n # Concrete annotation (same as ) form:\n annotation := ;\n\nDescription\n^^^^^^^^^^^\n\nThere are two forms of annotation declarations: abstract and concrete.\nThe *abstract annotation* form is used for declaring new kinds of\nannotation in a module. The *concrete annotation* declarations are\nused as sub-declarations for all other declarations in order to\nactually annotate them.\n\nThe annotation declaration options are as follows:\n\n:eql:synopsis:`abstract`\n If specified, the annotation will be *abstract*.\n\n:eql:synopsis:`inheritable`\n If specified, the annotation will be *inheritable*. The\n annotations are non-inheritable by default. That is, if a schema\n item has an annotation defined on it, the descendants of that\n schema item will not automatically inherit the annotation. Normal\n inheritance behavior can be turned on by declaring the annotation\n with the ``inheritable`` qualifier. This is only valid for *abstract\n annotation*.\n\n:eql:synopsis:``\n The name (optionally module-qualified) of the annotation.\n\n:eql:synopsis:``\n Any string value that the specified annotation is intended to have\n for the given context.\n\nThe only valid SDL sub-declarations are *concrete annotations*:\n\n:sdl:synopsis:``\n Annotations can also have annotations. Set the *annotation* of the\n enclosing annotation to a specific value.\n\n\n.. _ref_eql_ddl_annotations:\n\nDDL commands\n============\n\nThis section describes the low-level DDL commands for creating, altering,\nand dropping annotations and abstract annotations. You typically don't need to\nuse these commands directly, but knowing about them is useful for reviewing\nmigrations.\n\n\nCreate abstract annotation\n--------------------------\n\n:eql-statement:\n\nDefine a new annotation.\n\n.. eql:synopsis::\n\n [ with [, ...] ]\n create abstract [ inheritable ] annotation \n [\n \"{\"\n create annotation := ;\n [...]\n \"}\"\n ] ;\n\nDescription\n^^^^^^^^^^^\n\nThe command ``create abstract annotation`` defines a new annotation\nfor use in the current Gel database.\n\nIf *name* is qualified with a module name, then the annotation is created\nin that module, otherwise it is created in the current module.\nThe annotation name must be distinct from that of any existing schema item\nin the module.\n\nThe annotations are non-inheritable by default. That is, if a schema item\nhas an annotation defined on it, the descendants of that schema item will\nnot automatically inherit the annotation. Normal inheritance behavior can\nbe turned on by declaring the annotation with the ``inheritable`` qualifier.\n\nMost sub-commands and options of this command are identical to the\n:ref:`SDL annotation declaration `.\nThere's only one subcommand that is allowed in the ``create\nannotation`` block:\n\n:eql:synopsis:`create annotation := `\n Annotations can also have annotations. Set the\n :eql:synopsis:`` of the\n enclosing annotation to a specific :eql:synopsis:``.\n See :eql:stmt:`create annotation` for details.\n\nExample\n^^^^^^^\n\nDeclare an annotation ``extrainfo``:\n\n.. code-block:: edgeql\n\n create abstract annotation extrainfo;\n\n\nAlter abstract annotation\n-------------------------\n\n:eql-statement:\n\nChange the definition of an annotation.\n\n.. eql:synopsis::\n\n alter abstract annotation \n [ \"{\" ] ; [...] [ \"}\" ];\n\n # where is one of\n\n rename to \n create annotation := \n alter annotation := \n drop annotation \n\nDescription\n^^^^^^^^^^^\n\n:eql:synopsis:`alter abstract annotation` changes the definition of an\nabstract annotation.\n\nParameters\n^^^^^^^^^^\n\n:eql:synopsis:``\n The name (optionally module-qualified) of the annotation to alter.\n\nThe following subcommands are allowed in the ``alter abstract annotation``\nblock:\n\n:eql:synopsis:`rename to `\n Change the name of the annotation to :eql:synopsis:``.\n\n:eql:synopsis:`alter annotation := `\n Annotations can also have annotations. Change\n :eql:synopsis:`` to a specific\n :eql:synopsis:``. See :eql:stmt:`alter annotation` for\n details.\n\n:eql:synopsis:`drop annotation `\n Annotations can also have annotations. Remove annotation\n :eql:synopsis:``.\n See :eql:stmt:`drop annotation` for details.\n\nAll the subcommands allowed in the ``create abstract annotation``\nblock are also valid subcommands for ``alter annotation`` block.\n\nExample\n^^^^^^^\n\nRename an annotation:\n\n.. code-block:: edgeql\n\n alter abstract annotation extrainfo\n rename to extra_info;\n\n\nDrop abstract annotation\n------------------------\n\n:eql-statement:\n\nDrop a schema annotation.\n\n.. eql:synopsis::\n\n [ with [, ...] ]\n drop abstract annotation ;\n\nDescription\n^^^^^^^^^^^\n\nThe command ``drop abstract annotation`` removes an existing schema\nannotation from the database schema. Note that the ``inheritable``\nqualifier is not necessary in this statement.\n\nExample\n^^^^^^^\n\nDrop the annotation ``extra_info``:\n\n.. code-block:: edgeql\n\n drop abstract annotation extra_info;\n\n\nCreate annotation\n-----------------\n\n:eql-statement:\n\nDefine an annotation value for a given schema item.\n\n.. eql:synopsis::\n\n create annotation := \n\nDescription\n^^^^^^^^^^^\n\nThe command ``create annotation`` defines an annotation for a schema item.\n\n:eql:synopsis:`` refers to the name of a defined annotation,\nand :eql:synopsis:`` must be a constant EdgeQL expression\nevaluating into a string.\n\nThis statement can only be used as a subcommand in another\nDDL statement.\n\nExample\n^^^^^^^\n\nCreate an object type ``User`` and set its ``title`` annotation to\n``\"User type\"``.\n\n.. code-block:: edgeql\n\n create type User {\n create annotation title := \"User type\";\n };\n\n\nAlter annotation\n----------------\n\n:eql-statement:\n\nAlter an annotation value for a given schema item.\n\n.. eql:synopsis::\n\n alter annotation := \n\nDescription\n^^^^^^^^^^^\n\nThe command ``alter annotation`` alters an annotation value on a schema item.\n\n:eql:synopsis:`` refers to the name of a defined annotation,\nand :eql:synopsis:`` must be a constant EdgeQL expression\nevaluating into a string.\n\nThis statement can only be used as a subcommand in another\nDDL statement.\n\nExample\n^^^^^^^\n\nAlter an object type ``User`` and alter the value of its previously set\n``title`` annotation to ``\"User type\"``.\n\n.. code-block:: edgeql\n\n alter type User {\n alter annotation title := \"User type\";\n };\n\n\nDrop annotation\n---------------\n\n:eql-statement:\n\nRemove an annotation from a given schema item.\n\n.. eql:synopsis::\n\n drop annotation ;\n\nDescription\n^^^^^^^^^^^\n\nThe command ``drop annotation`` removes an annotation value from a schema item.\n\n:eql:synopsis:`` refers to the name of a defined annotation.\nThe annotation value does not have to exist on a schema item.\n\nThis statement can only be used as a subcommand in another\nDDL statement.\n\nExample\n^^^^^^^\n\nDrop the ``title`` annotation from the ``User`` object type:\n\n.. code-block:: edgeql\n\n alter type User {\n drop annotation title;\n };\n\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`Cheatsheets > Annotations `\n * - :ref:`Introspection > Object types `\n" }, { "path": "docs/reference/datamodel/branches.rst", "content": ".. _ref_datamodel_branches:\n.. _ref_datamodel_databases:\n\n.. versionadded:: 5.0\n\n========\nBranches\n========\n\nGel's |branches| are equivalent to PostgreSQL's *databases* and map to\nthem directly. Gel comes with tooling to help manage branches and build\na development workflow around them. E.g. when developing locally you can\nmap your Gel branches to your Git branches, and when using Gel Cloud and\nGitHub you can have a branch per PR.\n\n\nCLI commands\n============\n\nRefer to the :ref:`gel branch ` command group for\ndetails on the CLI commands for managing branches.\n\n\n.. _ref_admin_branches:\n\nDDL commands\n============\n\nThese are low-level commands that are used to create, alter, and drop branches.\nYou can use them when experimenting in REPL, of if you want to create your own\ntools to manage Gel branches.\n\n\nCreate empty branch\n-------------------\n\n:eql-statement:\n\nCreate a new branch without schema or data.\n\n.. eql:synopsis::\n\n create empty branch ;\n\nDescription\n^^^^^^^^^^^\n\nThe command ``create empty branch`` creates a new Gel branch without schema\nor data, aside from standard schemas.\n\nExample\n^^^^^^^\n\nCreate a new empty branch:\n\n.. code-block:: edgeql\n\n create empty branch newbranch;\n\n\nCreate schema branch\n--------------------\n\n:eql-statement:\n\nCreate a new branch copying the schema (without data)of an existing branch.\n\n.. eql:synopsis::\n\n create schema branch from ;\n\nDescription\n^^^^^^^^^^^\n\nThe command ``create schema branch`` creates a new Gel branch with schema\ncopied from an already existing branch.\n\nExample\n^^^^^^^\n\nCreate a new schema branch:\n\n.. code-block:: edgeql\n\n create schema branch feature from main;\n\n\nCreate data branch\n------------------\n\n:eql-statement:\n\nCreate a new branch copying the schema and data of an existing branch.\n\n.. eql:synopsis::\n\n create data branch from ;\n\nDescription\n^^^^^^^^^^^\n\nThe command ``create data branch`` creates a new Gel branch with schema and\ndata copied from an already existing branch.\n\nExample\n^^^^^^^\n\nCreate a new data branch:\n\n.. code-block:: edgeql\n\n create data branch feature from main;\n\n\nDrop branch\n-----------\n\n:eql-statement:\n\nRemove a branch.\n\n.. eql:synopsis::\n\n drop branch ;\n\nDescription\n^^^^^^^^^^^\n\nThe command ``drop branch`` removes an existing branch. It cannot be executed\nwhile there are existing connections to the target branch.\n\n.. warning::\n\n Executing ``drop branch`` removes data permanently and cannot be undone.\n\nExample\n^^^^^^^\n\nRemove a branch:\n\n.. code-block:: edgeql\n\n drop branch appdb;\n\n\nAlter branch\n------------\n\n:eql-statement:\n\nRename a branch.\n\n.. eql:synopsis::\n\n alter branch rename to ;\n\nDescription\n^^^^^^^^^^^\n\nThe command ``alter branch ā€¦ rename`` changes the name of an existing branch.\nIt cannot be executed while there are existing connections to the target\nbranch.\n\nExample\n^^^^^^^\n\nRename a branch:\n\n.. code-block:: edgeql\n\n alter branch featuer rename to feature;\n\nDatabase (deprecated)\n=====================\n\nVersions of Gel prior to 5.0 used the term *database* to refer to branches.\n\nCreate database\n---------------\n\n:eql-statement:\n\nCreate a new database.\n\n.. eql:synopsis::\n\n create database ;\n\nDescription\n^^^^^^^^^^^\n\nThe command ``create database`` creates a new Gel database.\n\nThe new database will be created with all standard schemas prepopulated.\n\nExamples\n^^^^^^^^\n\nCreate a new database:\n\n.. code-block:: edgeql\n\n create database appdb;\n\n\nDrop database\n-------------\n\n:eql-statement:\n\nRemove a database.\n\n.. eql:synopsis::\n\n drop database ;\n\nDescription\n^^^^^^^^^^^\n\nThe command ``drop database`` removes an existing database. It cannot\nbe executed while there are existing connections to the target\ndatabase.\n\n.. warning::\n\n Executing ``drop database`` removes data permanently and cannot be undone.\n\nExamples\n^^^^^^^^\n\nRemove a database:\n\n.. code-block:: edgeql\n\n drop database appdb;" }, { "path": "docs/reference/datamodel/comparison.rst", "content": ".. _ref_datamodel_comparison:\n\n===============\nvs SQL and ORMs\n===============\n\n|Gel's| approach to schema modeling builds upon the foundation of SQL while\ntaking cues from modern tools like ORM libraries. Let's see how it stacks up.\n\n.. _ref_datamodel_sql_comparison:\n\nComparison to SQL\n-----------------\n\nWhen using SQL databases, there's no convenient representation of the schema.\nInstead, the schema only exists as a series of ``{CREATE|ALTER|DELETE} {TABLE|\nCOLUMN}`` commands, usually spread across several SQL migration scripts.\nThere's no simple way to see the current state of your schema at a glance.\n\nMoreover, SQL stores data in a *relational* way. Connections between tables are\nrepresented with foreign key constraints and ``JOIN`` operations are required\nto query across tables.\n\n.. code-block::\n\n CREATE TABLE people (\n id uuid PRIMARY KEY,\n name text,\n );\n CREATE TABLE movies (\n id uuid PRIMARY KEY,\n title text,\n director_id uuid REFERENCES people(id)\n );\n\nIn |Gel|, connections between tables are represented with :ref:`Links\n`.\n\n.. code-block:: sdl\n\n type Movie {\n required title: str;\n required director: Person;\n }\n\n type Person {\n required name: str;\n }\n\nThis approach makes it simple to write queries that traverse this link, no\nJOINs required.\n\n.. code-block:: edgeql\n\n select Movie {\n title,\n director: {\n name\n }\n }\n\n.. _ref_datamodel_orm_comparison:\n\nComparison to ORMs\n------------------\n\nObject-relational mapping libraries are popular for a reason. They provide a\nway to model your schema and write queries in a way that feels natural in the\ncontext of modern, object-oriented programming languages. But ORMs have\ndownsides too.\n\n- **Lock-in**. Your schema is strongly coupled to the ORM library you are\n using. More generally, this also locks you into using a particular\n programming language.\n- Most ORMs have more **limited querying capabilities** than the query\n languages they abstract.\n- Many ORMs produce **suboptimal queries** that can have serious performance\n implications.\n- **Migrations** can be difficult. Since most ORMs aim to be the single source\n of truth for your schema, they necessarily must provide some sort of\n migration tool. These migration tools are maintained by the contributors to\n the ORM library, not the maintainers of the database itself. Quality control\n and long-term maintenance is not always guaranteed.\n\nFrom the beginning, Gel was designed to incorporate the best aspects of ORMs\nā€” declarative modeling, object-oriented APIs, and intuitive querying ā€”\nwithout the drawbacks.\n" }, { "path": "docs/reference/datamodel/computeds.rst", "content": ".. _ref_datamodel_computed:\n\n=========\nComputeds\n=========\n\n:edb-alt-title: Computed properties and links\n\n.. api-index:: :=\n\n.. important::\n\n This section assumes a basic understanding of EdgeQL. If you aren't familiar\n with it, feel free to skip this page for now.\n\nObject types can contain *computed* properties and links. Computed properties\nand links are not persisted in the database. Instead, they are evaluated *on\nthe fly* whenever that field is queried. Computed properties must be declared\nwith the ``property`` keyword and computed links must be declared with the\n``link`` keyword in |EdgeDB| versions prior to 4.0.\n\n\n.. code-block:: sdl\n\n type Person {\n name: str;\n all_caps_name := str_upper(__source__.name);\n }\n\nComputed fields are associated with an EdgeQL expression. This expression\ncan be an *arbitrary* EdgeQL query. This expression is evaluated whenever the\nfield is referenced in a query.\n\n.. note::\n\n Computed fields don't need to be pre-defined in your schema; you can drop\n them into individual queries as well. They behave in exactly the same way.\n For more information, see the :ref:`EdgeQL > Select > Computeds\n `.\n\n.. warning::\n\n :ref:`Volatile and modifying ` expressions are not\n allowed in computed properties defined in schema. This means that, for\n example, your schema-defined computed property cannot call\n :eql:func:`datetime_current`, but it *can* call\n :eql:func:`datetime_of_transaction` or :eql:func:`datetime_of_statement`.\n This does *not* apply to computed properties outside of schema.\n\n.. _ref_dot_notation:\n\nLeading dot notation\n--------------------\n\n.. api-index:: __source__\n\nThe example above used the special keyword ``__source__`` to refer to the\ncurrent object; it's analogous to ``this`` or ``self`` in many object-oriented\nlanguages.\n\nHowever, explicitly using ``__source__`` is optional here; inside the scope of\nan object type declaration, you can omit it entirely and use the ``.``\nshorthand.\n\n.. code-block:: sdl\n\n type Person {\n first_name: str;\n last_name: str;\n full_name := .first_name ++ ' ' ++ .last_name;\n }\n\nType and cardinality inference\n------------------------------\n\nThe type and cardinality of a computed field is *inferred* from the expression.\nThere's no need for the modifier keywords you use for non-computed fields (like\n``multi`` and ``required``). However, it's common to specify them anyway; it\nmakes the schema more readable and acts as a sanity check: if the provided\nEdgeQL expression disagrees with the modifiers, an error will be thrown the\nnext time you try to :ref:`create a migration `.\n\n.. code-block:: sdl\n\n type Person {\n first_name: str;\n\n # this is invalid, because first_name is not a required property\n required first_name_upper := str_upper(.first_name);\n }\n\nCommon use cases\n----------------\n\nFiltering\n^^^^^^^^^\n\nIf you find yourself writing the same ``filter`` expression repeatedly in\nqueries, consider defining a computed field that encapsulates the filter.\n\n.. code-block:: sdl\n\n type Club {\n multi members: Person;\n multi active_members := (\n select .members filter .is_active = true\n )\n }\n\n type Person {\n name: str;\n is_active: bool;\n }\n\n.. _ref_datamodel_links_backlinks:\n\nBacklinks\n^^^^^^^^^\n\nBacklinks are one of the most common use cases for computed links. In |Gel|\nlinks are *directional*; they have a source and a target. Often it's convenient\nto traverse a link in the *reverse* direction.\n\n.. code-block:: sdl\n\n type BlogPost {\n title: str;\n author: User;\n }\n\n type User {\n name: str;\n multi blog_posts := .`.\n\n\n.. list-table::\n :class: seealso\n\n * - :ref:`SDL > Links `\n * - :ref:`DDL > Links `\n * - :ref:`SDL > Properties `\n * - :ref:`DDL > Properties `\n" }, { "path": "docs/reference/datamodel/constraints.rst", "content": ".. _ref_datamodel_constraints:\n.. _ref_eql_sdl_constraints:\n\n===========\nConstraints\n===========\n\n.. index:: validation\n\nConstraints give users fine-grained control to ensure data consistency.\nThey can be defined on :ref:`properties `,\n:ref:`links`,\n:ref:`object types `,\nand :ref:`custom scalars `.\n\n\n.. _ref_datamodel_constraints_builtin:\n\nStandard constraints\n====================\n\n.. api-index:: exclusive, expression on, one_of, max_value, max_ex_value,\n min_value, min_ex_value, max_len_value, min_len_value, regexp\n\n|Gel| includes a number of standard ready-to-use constraints:\n\n.. include:: ../stdlib/constraint_table.rst\n\n\nConstraints on properties\n=========================\n\nExample: enforce all ``User`` objects to have a unique ``username``\nno longer than 25 characters:\n\n.. code-block:: sdl\n\n type User {\n required username: str {\n # usernames must be unique\n constraint exclusive;\n\n # max length (built-in)\n constraint max_len_value(25);\n };\n }\n\n.. _ref_datamodel_constraints_objects:\n\nConstraints on object types\n===========================\n\n.. api-index:: __subject__\n\nConstraints can be defined on object types. This is useful when the\nconstraint logic must reference multiple links or properties.\n\nExample: enforce that the magnitude of ``ConstrainedVector`` objects\nis no more than 5\n\n.. code-block:: sdl\n\n type ConstrainedVector {\n required x: float64;\n required y: float64;\n\n constraint expression on (\n (.x ^ 2 + .y ^ 2) ^ 0.5 <= 5\n # or, long form: `(__subject__.x + __subject__.y) ^ 0.5 <= 5`\n );\n }\n\nThe ``expression`` constraint is used here to define custom constraint logic.\nInside constraints, the keyword ``__subject__`` can be used to reference the\n*value* being constrained.\n\n.. note::\n Note that inside an object type declaration, you can omit ``__subject__``\n and simply refer to properties with the\n :ref:`leading dot notation ` (e.g. ``.property``).\n\n.. note::\n\n Also note that the constraint expression are fairly restricted. Due\n to how constraints are implemented, you can only reference ``single``\n (non-multi) properties and links defined on the object type:\n\n .. code-block:: sdl\n\n # Not valid!\n type User {\n required username: str;\n multi friends: User;\n\n # āŒ constraints cannot contain paths with more than one hop\n constraint expression on ('bob' in .friends.username);\n }\n\nAbstract constraints\n====================\n\n.. api-index:: abstract constraint\n\nYou can re-use constraints across multiple object types by declaring them as\nabstract constraints. Example:\n\n.. code-block:: sdl\n\n abstract constraint min_value(min: anytype) {\n errmessage :=\n 'Minimum allowed value for {__subject__} is {min}.';\n\n using (__subject__ >= min);\n }\n\n # use it like this:\n\n scalar type posint64 extending int64 {\n constraint min_value(0);\n }\n\n # or like this:\n\n type User {\n required age: int16 {\n constraint min_value(12);\n };\n }\n\n\nComputed constraints\n====================\n\nConstraints can be defined on computed properties:\n\n.. code-block:: sdl\n\n type User {\n required username: str;\n required clean_username := str_trim(str_lower(.username));\n\n constraint exclusive on (.clean_username);\n }\n\n\nComposite constraints\n=====================\n\n.. api-index:: constraint exclusive on\n\nTo define a composite constraint, create an ``exclusive`` constraint on a\ntuple of properties or links.\n\n.. code-block:: sdl\n\n type User {\n username: str;\n }\n\n type BlogPost {\n title: str;\n\n author: User;\n\n constraint exclusive on ((.title, .author));\n }\n\n.. _ref_datamodel_constraints_partial:\n\nPartial constraints\n===================\n\n.. api-index:: constraint exclusive on, except\n\nConstraints on object types can be made partial, so that they are not enforced\nwhen the specified ``except`` condition is met.\n\n.. code-block:: sdl\n\n type User {\n required username: str;\n deleted: bool;\n\n # Usernames must be unique unless marked deleted\n constraint exclusive on (.username) except (.deleted);\n }\n\n\nConstraints on links\n====================\n\nYou can constrain links such that a given object can only be linked once by\nusing :eql:constraint:`exclusive`:\n\n.. code-block:: sdl\n\n type User {\n required name: str;\n\n # Make sure none of the \"owned\" items belong\n # to any other user.\n multi owns: Item {\n constraint exclusive;\n }\n }\n\n\nLink property constraints\n=========================\n\nYou can also add constraints for :ref:`link properties\n`:\n\n.. code-block:: sdl\n\n type User {\n name: str;\n\n multi friends: User {\n strength: float64;\n\n constraint expression on (\n @strength >= 0\n );\n }\n }\n\n\nLink's \"@source\" and \"@target\"\n==============================\n\n.. api-index:: @source, @target\n\nYou can create a composite exclusive constraint on the object linking/linked\n*and* a link property by using ``@source`` or ``@target`` respectively. Here's\na schema for a library book management app that tracks books and who has\nchecked them out:\n\n.. code-block:: sdl\n\n type Book {\n required title: str;\n }\n\n type User {\n name: str;\n multi checked_out: Book {\n date: cal::local_date;\n\n # Ensures a given Book can be checked out\n # only once on a given day.\n constraint exclusive on ((@target, @date));\n }\n }\n\nHere, the constraint ensures that no book can be checked out to two ``User``\\s\non the same ``@date``.\n\nIn this example demonstrating ``@source``, we've created a schema to track\nplayer picks in a color-based memory game:\n\n.. code-block:: sdl\n\n type Player {\n required name: str;\n\n multi picks: Color {\n order: int16;\n\n constraint exclusive on ((@source, @order));\n }\n }\n\n type Color {\n required name: str;\n }\n\nThis constraint ensures that a single ``Player`` cannot pick two ``Color``\\s at\nthe same ``@order``.\n\n\nConstraints on custom scalars\n=============================\n\nCustom scalar types can be constrained.\n\n.. code-block:: sdl\n\n scalar type username extending str {\n constraint regexp(r'^[A-Za-z0-9_]{4,20}$');\n }\n\nNote: you can't use :eql:constraint:`exclusive` constraints on custom scalar\ntypes, as the concept of exclusivity is only defined in the context of a given\nobject type.\n\nUse :eql:constraint:`expression` constraints to declare custom constraints\nusing arbitrary EdgeQL expressions. The example below uses the built-in\n:eql:func:`str_trim` function.\n\n.. code-block:: sdl\n\n scalar type title extending str {\n constraint expression on (\n __subject__ = str_trim(__subject__)\n );\n }\n\n\nConstraints and inheritance\n===========================\n\n.. api-index:: delegated constraint\n\nIf you define a constraint on a type and then extend that type, the constraint\nwill *not* be applied individually to each extending type. Instead, it will\napply globally across all the types that inherited the constraint.\n\n.. code-block:: sdl\n\n type User {\n required name: str {\n constraint exclusive;\n }\n }\n type Administrator extending User;\n type Moderator extending User;\n\n.. code-block:: edgeql-repl\n\n gel> insert Administrator {\n .... name := 'Jan'\n .... };\n {default::Administrator {id: 7aeaa146-f5a5-11ed-a598-53ddff476532}}\n\n gel> insert Moderator {\n .... name := 'Jan'\n .... };\n gel error: ConstraintViolationError: name violates exclusivity constraint\n Detail: value of property 'name' of object type 'default::Moderator'\n violates exclusivity constraint\n\n gel> insert User {\n .... name := 'Jan'\n .... };\n gel error: ConstraintViolationError: name violates exclusivity constraint\n Detail: value of property 'name' of object type 'default::User'\n violates exclusivity constraint\n\nAs this example demonstrates, if an object of one extending type has a value\nfor a property that is exclusive, an object of a *different* extending type\ncannot have the same value.\n\nIf that's not what you want, you can instead delegate the constraint to the\ninheriting types by prepending the ``delegated`` keyword to the constraint.\nThe constraint would then be applied just as if it were declared individually\non each of the inheriting types.\n\n.. code-block:: sdl\n\n type User {\n required name: str {\n delegated constraint exclusive;\n }\n }\n type Administrator extending User;\n type Moderator extending User;\n\n.. code-block:: edgeql-repl\n\n gel> insert Administrator {\n .... name := 'Jan'\n .... };\n {default::Administrator {id: 7aeaa146-f5a5-11ed-a598-53ddff476532}}\n\n gel> insert User {\n .... name := 'Jan'\n .... };\n {default::User {id: a6e3fdaf-c44b-4080-b39f-6a07496de66b}}\n\n gel> insert Moderator {\n .... name := 'Jan'\n .... };\n {default::Moderator {id: d3012a3f-0f16-40a8-8884-7203f393b63d}}\n\n gel> insert Moderator {\n .... name := 'Jan'\n .... };\n gel error: ConstraintViolationError: name violates exclusivity constraint\n Detail: value of property 'name' of object type 'default::Moderator'\n violates exclusivity constraint\n\nWith the addition of ``delegated`` to the constraints, the inserts were\nsuccessful for each of the types. We did not hit a constraint violation\nuntil we tried to insert a second ``Moderator`` object with the same\nname as the existing one.\n\n\n.. _ref_eql_sdl_constraints_syntax:\n\nDeclaring constraints\n=====================\n\nThis section describes the syntax to declare constraints in your schema.\n\nSyntax\n------\n\n.. sdl:synopsis::\n\n [{abstract | delegated}] constraint [ ( [] [, ...] ) ]\n [ on ( ) ]\n [ except ( ) ]\n [ extending [, ...] ]\n \"{\"\n [ using ; ]\n [ errmessage := ; ]\n [ ]\n [ ... ]\n \"}\" ;\n\n # where is:\n\n [ : ] { | }\n\nDescription\n^^^^^^^^^^^\n\nThis declaration defines a new constraint with the following options:\n\n:eql:synopsis:`abstract`\n If specified, the constraint will be *abstract*.\n\n:eql:synopsis:`delegated`\n If specified, the constraint is defined as *delegated*, which means\n that it will not be enforced on the type it's declared on, and the\n enforcement will be delegated to the subtypes of this type.\n This is particularly useful for :eql:constraint:`exclusive`\n constraints in abstract types. This is only valid for *concrete\n constraints*.\n\n:eql:synopsis:``\n The name (optionally module-qualified) of the new constraint.\n\n:eql:synopsis:``\n An optional list of constraint arguments.\n\n For an *abstract constraint* :eql:synopsis:`` optionally\n specifies the argument name and :eql:synopsis:`` specifies\n the argument type.\n\n For a *concrete constraint* :eql:synopsis:`` optionally\n specifies the argument name and :eql:synopsis:`` specifies\n the argument value. The argument value specification must match the\n parameter declaration of the abstract constraint.\n\n:eql:synopsis:`on ( )`\n An optional expression defining the *subject* of the constraint.\n If not specified, the subject is the value of the schema item on which\n the concrete constraint is defined.\n\n The expression must refer to the original subject of the constraint as\n ``__subject__``. The expression must be\n :ref:`Immutable `, but may refer to\n ``__subject__`` and its properties and links.\n\n Note also that ```` itself has to\n be parenthesized.\n\n:eql:synopsis:`except ( )`\n An optional expression defining a condition to create exceptions to\n the constraint. If ```` evaluates to ``true``,\n the constraint is ignored for the current subject. If it evaluates\n to ``false`` or ``{}``, the constraint applies normally.\n\n ``except`` may only be declared on object constraints, and otherwise\n follows the same rules as ``on``.\n\n:eql:synopsis:`extending [, ...]`\n If specified, declares the *parent* constraints for this abstract\n constraint.\n\nThe valid SDL sub-declarations are listed below:\n\n:eql:synopsis:`using `\n A boolean expression that returns ``true`` for valid data and\n ``false`` for invalid data. The expression may refer to the\n subject of the constraint as ``__subject__``. This declaration is\n only valid for *abstract constraints*.\n\n:eql:synopsis:`errmessage := `\n An optional string literal defining the error message template\n that is raised when the constraint is violated. The template is a\n formatted string that may refer to constraint context variables in\n curly braces. The template may refer to the following:\n\n - ``$argname`` -- the value of the specified constraint argument\n - ``__subject__`` -- the value of the ``title`` annotation of the\n scalar type, property or link on which the constraint is defined.\n\n If the content of curly braces does not match any variables,\n the curly braces are emitted as-is. They can also be escaped by\n using double curly braces.\n\n:sdl:synopsis:``\n Set constraint :ref:`annotation `\n to a given *value*.\n\n\n.. _ref_eql_ddl_constraints:\n\nDDL commands\n============\n\nThis section describes the low-level DDL commands for creating and dropping\nconstraints and abstract constraints. You typically don't need to use these\ncommands directly, but knowing about them is useful for reviewing migrations.\n\n\nCreate abstract constraint\n--------------------------\n\n:eql-statement:\n:eql-haswith:\n\nDefine a new abstract constraint.\n\n.. eql:synopsis::\n\n [ with [ := ] module ]\n create abstract constraint [ ( [] [, ...] ) ]\n [ on ( ) ]\n [ extending [, ...] ]\n \"{\" ; [...] \"}\" ;\n\n # where is:\n\n [ : ] \n\n # where is one of\n\n using \n set errmessage := \n create annotation := \n\n\nDescription\n^^^^^^^^^^^\nThe command ``create abstract constraint`` defines a new abstract constraint.\n\nIf *name* is qualified with a module name, then the constraint is created in\nthat module, otherwise it is created in the current module. The constraint\nname must be distinct from that of any existing schema item in the module.\n\n\nParameters\n^^^^^^^^^^\nMost sub-commands and options of this command are identical to the\n:ref:`SDL constraint declaration `,\nwith some additional features listed below:\n\n:eql:synopsis:`[ := ] module `\n An optional list of module alias declarations to be used in the\n migration definition. When *module-alias* is not specified,\n *module-name* becomes the effective current module and is used\n to resolve all unqualified names.\n\n:eql:synopsis:`set errmessage := `\n An optional string literal defining the error message template\n that is raised when the constraint is violated. Other than a\n slight syntactical difference this is the same as the\n corresponding SDL declaration.\n\n:eql:synopsis:`create annotation := ;`\n Set constraint annotation ```` to ````.\n See :eql:stmt:`create annotation` for details.\n\n\nExample\n^^^^^^^\nCreate an abstract constraint \"uppercase\" which checks if the subject\nis a string in upper case:\n\n.. code-block:: edgeql\n\n create abstract constraint uppercase {\n create annotation title := \"Upper case constraint\";\n\n using (str_upper(__subject__) = __subject__);\n\n set errmessage := \"{__subject__} is not in upper case\";\n };\n\n\nAlter abstract constraint\n-------------------------\n\n:eql-statement:\n:eql-haswith:\n\nAlter the definition of an abstract constraint.\n\n.. eql:synopsis::\n\n [ with [ := ] module ]\n alter abstract constraint \n \"{\" ; [...] \"}\" ;\n\n # where is one of\n\n rename to \n using \n set errmessage := \n reset errmessage\n create annotation := \n alter annotation := \n drop annotation \n\n\nDescription\n^^^^^^^^^^^\n\nThe command ``alter abstract constraint`` changes the definition of an\nabstract constraint item. *name* must be a name of an existing\nabstract constraint, optionally qualified with a module name.\n\n\nParameters\n^^^^^^^^^^\n\n:eql:synopsis:`[ := ] module `\n An optional list of module alias declarations to be used in the\n migration definition. When *module-alias* is not specified,\n *module-name* becomes the effective current module and is used\n to resolve all unqualified names.\n\n:eql:synopsis:``\n The name (optionally module-qualified) of the constraint to alter.\n\nSubcommands allowed in the ``alter abstract constraint`` block:\n\n:eql:synopsis:`rename to `\n Change the name of the constraint to *newname*. All concrete\n constraints inheriting from this constraint are also renamed.\n\n:eql:synopsis:`alter annotation := `\n Alter constraint annotation ````.\n See :eql:stmt:`alter annotation` for details.\n\n:eql:synopsis:`drop annotation `\n Remove annotation ````.\n See :eql:stmt:`drop annotation` for details.\n\n:eql:synopsis:`reset errmessage`\n Remove the error message from this abstract constraint. The error message\n specified in the base abstract constraint will be used instead.\n\nAll subcommands allowed in a ``create abstract constraint`` block are also\nvalid here.\n\n\nExample\n^^^^^^^\n\nRename the abstract constraint \"uppercase\" to \"upper_case\":\n\n.. code-block:: edgeql\n\n alter abstract constraint uppercase rename to upper_case;\n\n\nDrop abstract constraint\n------------------------\n\n:eql-statement:\n:eql-haswith:\n\nRemove an abstract constraint from the schema.\n\n.. eql:synopsis::\n\n [ with [ := ] module ]\n drop abstract constraint ;\n\n\nDescription\n^^^^^^^^^^^\n\nThe command ``drop abstract constraint`` removes an existing abstract\nconstraint item from the database schema. If any schema items depending\non this constraint exist, the operation is refused.\n\n\nParameters\n^^^^^^^^^^\n\n:eql:synopsis:`[ := ] module `\n An optional list of module alias declarations to be used in the\n migration definition.\n\n:eql:synopsis:``\n The name (optionally module-qualified) of the constraint to remove.\n\n\nExample\n^^^^^^^\n\nDrop abstract constraint ``upper_case``:\n\n.. code-block:: edgeql\n\n drop abstract constraint upper_case;\n\n\nCreate constraint\n-----------------\n\n:eql-statement:\n\nDefine a concrete constraint on the specified schema item.\n\n.. eql:synopsis::\n\n [ with [ := ] module ]\n create [ delegated ] constraint \n [ ( [] [, ...] ) ]\n [ on ( ) ]\n [ except ( ) ]\n \"{\" ; [...] \"}\" ;\n\n # where is:\n\n [ : ] \n\n # where is one of\n\n set errmessage := \n create annotation := \n\n\nDescription\n^^^^^^^^^^^\n\nThe command ``create constraint`` defines a new concrete constraint. It can\nonly be used in the context of :eql:stmt:`create scalar`,\n:eql:stmt:`alter scalar`, :eql:stmt:`create property`,\n:eql:stmt:`alter property`, :eql:stmt:`create link`, or :eql:stmt:`alter link`.\n\n*name* must be a name (optionally module-qualified) of a previously defined\nabstract constraint.\n\n\nParameters\n^^^^^^^^^^\n\nMost sub-commands and options of this command are identical to the\n:ref:`SDL constraint declaration `,\nwith some additional features listed below:\n\n:eql:synopsis:`[ := ] module `\n An optional list of module alias declarations to be used in the\n migration definition.\n\n:eql:synopsis:`set errmessage := `\n An optional string literal defining the error message template\n that is raised when the constraint is violated. Other than a\n slight syntactical difference, this is the same as the corresponding\n SDL declaration.\n\n:eql:synopsis:`create annotation := ;`\n An optional list of annotations for the constraint. See\n :eql:stmt:`create annotation` for details.\n\n\nExample\n^^^^^^^\n\nCreate a \"score\" property on the \"User\" type with a minimum value\nconstraint:\n\n.. code-block:: edgeql\n\n alter type User create property score: int64 {\n create constraint min_value(0)\n };\n\nCreate a Vector with a maximum magnitude:\n\n.. code-block:: edgeql\n\n create type Vector {\n create required property x: float64;\n create required property y: float64;\n create constraint expression ON (\n __subject__.x^2 + __subject__.y^2 < 25\n );\n }\n\n\nAlter constraint\n----------------\n\n:eql-statement:\n\nAlter the definition of a concrete constraint on the specified schema item.\n\n.. eql:synopsis::\n\n [ with [ := ] module [, ...] ]\n alter constraint \n [ ( [] [, ...] ) ]\n [ on ( ) ]\n [ except ( ) ]\n \"{\" ; [ ... ] \"}\" ;\n\n # -- or --\n\n [ with [ := ] module [, ...] ]\n alter constraint \n [ ( [] [, ...] ) ]\n [ on ( ) ]\n ;\n\n # where is one of:\n\n set delegated\n set not delegated\n set errmessage := \n reset errmessage\n create annotation := \n alter annotation \n drop annotation \n\n\nDescription\n^^^^^^^^^^^\n\nThe command ``alter constraint`` changes the definition of a concrete\nconstraint. Both single- and multi-command forms are supported.\n\n\nParameters\n^^^^^^^^^^\n\n:eql:synopsis:`[ := ] module `\n An optional list of module alias declarations for the migration.\n\n:eql:synopsis:``\n The name (optionally module-qualified) of the concrete constraint\n that is being altered.\n\n:eql:synopsis:``\n A list of constraint arguments as specified at the time of\n ``create constraint``.\n\n:eql:synopsis:`on ( )`\n An expression defining the *subject* of the constraint as specified\n at the time of ``create constraint``.\n\nThe following subcommands are allowed in the ``alter constraint`` block:\n\n:eql:synopsis:`set delegated`\n Mark the constraint as *delegated*, which means it will\n not be enforced on the type it's declared on, and enforcement is\n delegated to subtypes. Useful for :eql:constraint:`exclusive` constraints.\n\n:eql:synopsis:`set not delegated`\n Mark the constraint as *not delegated*, so it is enforced globally across\n the type and any extending types.\n\n:eql:synopsis:`rename to `\n Change the name of the constraint to ````.\n\n:eql:synopsis:`alter annotation `\n Alter a constraint annotation.\n\n:eql:synopsis:`drop annotation `\n Remove a constraint annotation.\n\n:eql:synopsis:`reset errmessage`\n Remove the error message from this constraint, reverting to that of the\n abstract constraint, if any.\n\nAll subcommands allowed in ``create constraint`` are also valid in\n``alter constraint``.\n\nExample\n^^^^^^^\n\nChange the error message on the minimum value constraint on the property\n\"score\" of the \"User\" type:\n\n.. code-block:: edgeql\n\n alter type User alter property score\n alter constraint min_value(0)\n set errmessage := 'Score cannot be negative';\n\n\nDrop constraint\n---------------\n\n:eql-statement:\n:eql-haswith:\n\nRemove a concrete constraint from the specified schema item.\n\n.. eql:synopsis::\n\n [ with [ := ] module [, ...] ]\n drop constraint \n [ ( [] [, ...] ) ]\n [ on ( ) ]\n [ except ( ) ] ;\n\n\nDescription\n^^^^^^^^^^^\n\nThe command ``drop constraint`` removes the specified constraint from\nits containing schema item.\n\n\nParameters\n^^^^^^^^^^\n\n:eql:synopsis:`[ := ] module `\n Optional module alias declarations for the migration definition.\n\n:eql:synopsis:``\n The name (optionally module-qualified) of the concrete constraint\n to remove.\n\n:eql:synopsis:``\n A list of constraint arguments as specified at the time of\n ``create constraint``.\n\n:eql:synopsis:`on ( )`\n Expression defining the *subject* of the constraint as specified\n at the time of ``create constraint``.\n\nExample\n^^^^^^^\n\nRemove constraint \"min_value\" from the property \"score\" of the \"User\" type:\n\n.. code-block:: edgeql\n\n alter type User alter property score\n drop constraint min_value(0);\n\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`Introspection > Constraints `\n * - :ref:`Standard Library > Constraints `\n" }, { "path": "docs/reference/datamodel/extensions.rst", "content": ".. _ref_datamodel_extensions:\n\n==========\nExtensions\n==========\n\n.. api-index:: using extension\n\nExtensions are the way |Gel| can be extended with more functionality.\nThey can add new types, scalars, functions, etc., but, more\nimportantly, they can add new ways of interacting with the database.\n\n\nBuilt-in extensions\n===================\n\n.. api-index:: edgeql_http, graphql, auth, ai, pg_trgm, pg_unaccent, pgcrypto,\n pgvector\n\nThere are a few built-in extensions available:\n\n- ``edgeql_http``: enables :ref:`EdgeQL over HTTP `,\n- ``graphql``: enables :ref:`GraphQL `,\n- ``auth``: enables :ref:`Gel Auth `,\n- ``ai``: enables :ref:`ext::ai module `,\n\n- ``pg_trgm``: enables ``ext::pg_trgm``, which re-exports\n `pgtrgm `__,\n\n- ``pg_unaccent``: enables ``ext::pg_unaccent``, which re-exports\n `unaccent `__,\n\n- ``pgcrypto``: enables ``ext::pgcrypto``, which re-exports\n `pgcrypto `__,\n\n- ``pgvector``: enables ``ext::pgvector``, which re-exports\n `pgvector `__,\n\n.. _ref_datamodel_using_extension:\n\nTo enable these extensions, add a ``using`` statement at the top level of\nyour schema:\n\n.. code-block:: sdl\n\n using extension auth;\n # or / and\n using extension ai;\n\n\nStandalone extensions\n=====================\n\n.. api-index:: postgis\n\nAdditionally, standalone extension packages can be installed on local project-managed instances via the CLI, with ``postgis`` being a notable example.\n\nList installed extensions:\n\n.. code-block:: bash\n\n $ gel extension list\n ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”\n ā”‚ Name ā”‚ Version ā”‚\n ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜\n\nList available extensions:\n\n.. code-block:: bash\n\n $ gel extension list-available\n ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”\n ā”‚ Name ā”‚ Version ā”‚\n ā”‚ postgis ā”‚ 3.4.3+6b82d77 ā”‚\n ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜\n\nInstall the ``postgis`` extension:\n\n.. code-block:: bash\n\n $ gel extension install postgis\n Found extension package: postgis version 3.4.3+6b82d77\n 00:00:03 [====================] 22.49 MiB/22.49 MiB\n Extension 'postgis' installed successfully.\n\nCheck that extension is installed:\n\n.. code-block:: bash\n\n $ gel extension list\n ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”\n ā”‚ Name ā”‚ Version ā”‚\n ā”‚ postgis ā”‚ 3.4.3+6b82d77 ā”‚\n ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜\n\nAfter installing extensions, make sure to restart your instance:\n\n.. code-block:: bash\n\n $ gel instance restart\n\nStandalone extensions can now be declared in the schema, same as\nbuilt-in extensions:\n\n.. code-block:: sdl\n\n using extension postgis;\n\n.. note::\n To restore a dump that uses a standalone extension, that extension must\n be installed before the restore process.\n\n.. _ref_eql_sdl_extensions:\n\nUsing extensions\n================\n\nSyntax\n------\n\n.. sdl:synopsis::\n\n using extension \";\"\n\n\nExtension declaration must be outside any :ref:`module block\n` since extensions affect the entire database and\nnot a specific module.\n\n\n\n.. _ref_eql_ddl_extensions:\n\nDDL commands\n============\n\nThis section describes the low-level DDL commands for creating and\ndropping extensions. You typically don't need to use these commands directly,\nbut knowing about them is useful for reviewing migrations.\n\n\ncreate extension\n----------------\n\n:eql-statement:\n\nEnable a particular extension for the current schema.\n\n.. eql:synopsis::\n\n create extension \";\"\n\n\nDescription\n^^^^^^^^^^^\n\nThe command ``create extension`` enables the specified extension for\nthe current :versionreplace:`database;5.0:branch`.\n\nExamples\n^^^^^^^^\n\nEnable :ref:`GraphQL ` extension for the current\nschema:\n\n.. code-block:: edgeql\n\n create extension graphql;\n\nEnable :ref:`EdgeQL over HTTP ` extension for the\ncurrent :versionreplace:`database;5.0:branch`:\n\n.. code-block:: edgeql\n\n create extension edgeql_http;\n\n\ndrop extension\n--------------\n\n:eql-statement:\n\nDisable an extension.\n\n.. eql:synopsis::\n\n drop extension \";\"\n\n\nThe command ``drop extension`` disables a currently active extension for\nthe current |branch|.\n\nExamples\n^^^^^^^^\n\nDisable :ref:`GraphQL ` extension for the current\nschema:\n\n.. code-block:: edgeql\n\n drop extension graphql;\n\nDisable :ref:`EdgeQL over HTTP ` extension for the\ncurrent :versionreplace:`database;5.0:branch`:\n\n.. code-block:: edgeql\n\n drop extension edgeql_http;\n" }, { "path": "docs/reference/datamodel/functions.rst", "content": ".. _ref_datamodel_functions:\n.. _ref_eql_sdl_functions:\n\n=========\nFunctions\n=========\n\n.. note::\n\n This page documents how to define custom functions, however |Gel| provides a\n large library of built-in functions and operators. These are documented in\n :ref:`Standard Library `.\n\n\nUser-defined Functions\n======================\n\nGel allows you to define custom functions. For example, consider\na function that adds an exclamation mark ``'!'`` at the end of the\nstring:\n\n.. code-block:: sdl\n\n function exclamation(word: str) -> str\n using (word ++ '!');\n\nThis function accepts a :eql:type:`str` as an argument and produces a\n:eql:type:`str` as output as well.\n\n.. code-block:: edgeql-repl\n\n test> select exclamation({'Hello', 'World'});\n {'Hello!', 'World!'}\n\n\n.. _ref_datamodel_functions_modifying:\n\nSets as arguments\n=================\n\nCalling a user-defined function on a set will always apply it as\n:ref:`*element-wise* `.\n\n.. code-block:: sdl\n\n function magnitude(x: float64) -> float64\n using (\n math::sqrt(sum(x * x))\n );\n\n.. code-block:: edgeql-repl\n\n db> select magnitude({3, 4});\n {3, 4}\n\nIn order to pass in multiple arguments at once, arguments should be packed into\narrays:\n\n.. code-block:: sdl\n\n function magnitude(xs: array) -> float64\n using (\n with x := array_unpack(xs)\n select math::sqrt(sum(x * x))\n );\n\n.. code-block:: edgeql-repl\n\n db> select magnitude([3, 4]);\n {5}\n\nMultiple packed arrays can be passed into such a function, which will then be\napplied element-wise.\n\n.. code-block:: edgeql-repl\n\n db> select magnitude({[3, 4], [5, 12]});\n {5, 13}\n\n\nModifying Functions\n===================\n\n.. versionadded:: 6.0\n\nUser-defined functions can contain DML (i.e.,\n:ref:`insert `, :ref:`update `,\n:ref:`delete `) to make changes to existing data. These\nfunctions have a :ref:`modifying ` volatility.\n\n.. code-block:: sdl\n\n function add_user(name: str) -> User\n using (\n insert User {\n name := name,\n joined_at := std::datetime_current(),\n }\n );\n\n.. code-block:: edgeql-repl\n\n db> select add_user('Jan') {name, joined_at};\n {default::User {name: 'Jan', joined_at: '2024-12-11T11:49:47Z'}}\n\nUnlike other functions, the arguments of modifying functions **must** have a\n:ref:`cardinality ` of ``One``.\n\n.. code-block:: edgeql-repl\n\n db> select add_user({'Feb','Mar'});\n gel error: QueryError: possibly more than one element passed into\n modifying function\n db> select add_user({});\n gel error: QueryError: possibly an empty set passed as non-optional\n argument into modifying function\n\nOptional arguments can still accept empty sets. For example, if ``add_user``\nwas defined as:\n\n.. code-block:: sdl\n\n function add_user(name: str, joined_at: optional datetime) -> User\n using (\n insert User {\n name := name,\n joined_at := joined_at ?? std::datetime_current(),\n }\n );\n\nthen the following queries are valid:\n\n.. code-block:: edgeql-repl\n\n db> select add_user('Apr', {}) {name, joined_at};\n {default::User {name: 'Apr', joined_at: '2024-12-11T11:50:51Z'}}\n db> select add_user('May', '2024-12-11T12:00:00-07:00') {name, joined_at};\n {default::User {name: 'May', joined_at: '2024-12-11T12:00:00Z'}}\n\nIn order to insert or update a multi parameter, the desired arguments should be\naggregated into an array as described above:\n\n.. code-block:: sdl\n\n function add_user(name: str, nicknames: array) -> User\n using (\n insert User {\n name := name,\n nicknames := array_unpack(nicknames),\n }\n );\n\n\n.. _ref_eql_sdl_functions_syntax:\n\nDeclaring functions\n===================\n\n.. api-index:: function, using, ->, variadic, named only, set of, optional,\n volatility, Immutable, Stable, Volatile, Modifying\n\nThis section describes the syntax to declare a function in your schema.\n\nSyntax\n------\n\n.. sdl:synopsis::\n\n function ([ ] [, ... ]) -> \n using ( );\n\n function ([ ] [, ... ]) -> \n using ;\n\n function ([ ] [, ... ]) -> \n \"{\"\n [ ]\n [ volatility := {'Immutable' | 'Stable' | 'Volatile' | 'Modifying'} ]\n [ using ( ) ; ]\n [ using ; ]\n [ ... ]\n \"}\" ;\n\n # where is:\n\n [ ] : [ ] [ = ]\n\n # is:\n\n [ { variadic | named only } ]\n\n # is:\n\n [ { set of | optional } ]\n\n # and is:\n\n [ ] \n\n\nDescription\n^^^^^^^^^^^\n\nThis declaration defines a new **function** with the following options:\n\n:eql:synopsis:``\n The name (optionally module-qualified) of the function to create.\n\n:eql:synopsis:``\n The kind of an argument: ``variadic`` or ``named only``.\n\n If not specified, the argument is called *positional*.\n\n The ``variadic`` modifier indicates that the function takes an\n arbitrary number of arguments of the specified type. The passed\n arguments will be passed as an array of the argument type.\n Positional arguments cannot follow a ``variadic`` argument.\n ``variadic`` parameters cannot have a default value.\n\n The ``named only`` modifier indicates that the argument can only\n be passed using that specific name. Positional arguments cannot\n follow a ``named only`` argument.\n\n:eql:synopsis:``\n The name of an argument. If ``named only`` modifier is used this\n argument *must* be passed using this name only.\n\n.. _ref_sdl_function_typequal:\n\n:eql:synopsis:``\n The type qualifier: ``set of`` or ``optional``.\n\n The ``set of`` qualifier indicates that the function is taking the\n argument as a *whole set*, as opposed to being called on the input\n product element-by-element.\n\n User defined functions can not use ``set of`` arguments.\n\n The ``optional`` qualifier indicates that the function will be called\n if the argument is an empty set. The default behavior is to return\n an empty set if the argument is not marked as ``optional``.\n\n:eql:synopsis:``\n The data type of the function's arguments\n (optionally module-qualified).\n\n:eql:synopsis:``\n An expression to be used as default value if the parameter is not\n specified. The expression has to be of a type compatible with the\n type of the argument.\n\n.. _ref_sdl_function_rettype:\n\n:eql:synopsis:``\n The return data type (optionally module-qualified).\n\n The ``set of`` modifier indicates that the function will return\n a non-singleton set.\n\n The ``optional`` qualifier indicates that the function may return\n an empty set.\n\nThe valid SDL sub-declarations are listed below:\n\n:eql:synopsis:`volatility := {'Immutable' | 'Stable' | 'Volatile' | 'Modifying'}`\n Function volatility determines how aggressively the compiler can\n optimize its invocations.\n\n If not explicitly specified the function volatility is\n :ref:`inferred ` from the function body.\n\n * An ``Immutable`` function cannot modify the database and is\n guaranteed to return the same results given the same arguments\n *in all statements*.\n\n * A ``Stable`` function cannot modify the database and is\n guaranteed to return the same results given the same\n arguments *within a single statement*.\n\n * A ``Volatile`` function cannot modify the database and can return\n different results on successive calls with the same arguments.\n\n * A ``Modifying`` function can modify the database and can return\n different results on successive calls with the same arguments.\n\n:eql:synopsis:`using ( )`\n Specifies the body of the function. :eql:synopsis:`` is an\n arbitrary EdgeQL expression.\n\n:eql:synopsis:`using `\n A verbose version of the :eql:synopsis:`using` clause that allows\n specifying the language of the function body.\n\n * :eql:synopsis:`` is the name of the language that\n the function is implemented in. Currently can only be ``edgeql``.\n\n * :eql:synopsis:`` is a string constant defining\n the function. It is often helpful to use\n :ref:`dollar quoting `\n to write the function definition string.\n\n:sdl:synopsis:``\n Set function :ref:`annotation `\n to a given *value*.\n\nThe function name must be distinct from that of any existing function\nwith the same argument types in the same module. Functions of\ndifferent argument types can share a name, in which case the functions\nare called *overloaded functions*.\n\n\n.. _ref_eql_ddl_functions:\n\nDDL commands\n============\n\nThis section describes the low-level DDL commands for creating, altering, and\ndropping functions. You typically don't need to use these commands directly, but\nknowing about them is useful for reviewing migrations.\n\nCreate function\n---------------\n\n:eql-statement:\n:eql-haswith:\n\nDefine a new function.\n\n.. eql:synopsis::\n\n [ with [, ...] ]\n create function ([ ] [, ... ]) -> \n using ( );\n\n [ with [, ...] ]\n create function ([ ] [, ... ]) -> \n using ;\n\n [ with [, ...] ]\n create function ([ ] [, ... ]) -> \n \"{\" [, ...] \"}\" ;\n\n # where is:\n\n [ ] : [ ] [ = ]\n\n # is:\n\n [ { variadic | named only } ]\n\n # is:\n\n [ { set of | optional } ]\n\n # and is:\n\n [ ] \n\n # and is one of\n\n set volatility := {'Immutable' | 'Stable' | 'Volatile' | 'Modifying'} ;\n create annotation := ;\n using ( ) ;\n using ;\n\n\nDescription\n^^^^^^^^^^^\n\nThe command ``create function`` defines a new function. If *name* is\nqualified with a module name, then the function is created in that\nmodule, otherwise it is created in the current module.\n\nThe function name must be distinct from that of any existing function\nwith the same argument types in the same module. Functions of\ndifferent argument types can share a name, in which case the functions\nare called *overloaded functions*.\n\n\nParameters\n^^^^^^^^^^\n\nMost sub-commands and options of this command are identical to the\n:ref:`SDL function declaration `, with\nsome additional features listed below:\n\n:eql:synopsis:`set volatility := {'Immutable' | 'Stable' | 'Volatile' | 'Modifying'}`\n Function volatility determines how aggressively the compiler can\n optimize its invocations. Other than a slight syntactical\n difference this is the same as the corresponding SDL declaration.\n\n:eql:synopsis:`create annotation := `\n Set the function's :eql:synopsis:`` to\n :eql:synopsis:``.\n\n See :eql:stmt:`create annotation` for details.\n\n\nExamples\n^^^^^^^^\n\nDefine a function returning the sum of its arguments:\n\n.. code-block:: edgeql\n\n create function mysum(a: int64, b: int64) -> int64\n using (\n select a + b\n );\n\nThe same, but using a variadic argument and an explicit language:\n\n.. code-block:: edgeql\n\n create function mysum(variadic argv: int64) -> int64\n using edgeql $$\n select sum(array_unpack(argv))\n $$;\n\nDefine a function using the block syntax:\n\n.. code-block:: edgeql\n\n create function mysum(a: int64, b: int64) -> int64 {\n using (\n select a + b\n );\n create annotation title := \"My sum function.\";\n };\n\n\nAlter function\n--------------\n\n:eql-statement:\n:eql-haswith:\n\nChange the definition of a function.\n\n.. eql:synopsis::\n\n [ with [, ...] ]\n alter function ([ ] [, ... ]) \"{\"\n [, ...]\n \"}\"\n\n # where is:\n\n [ ] : [ ] [ = ]\n\n # and is one of\n\n set volatility := {'Immutable' | 'Stable' | 'Volatile' | 'Modifying'} ;\n reset volatility ;\n rename to ;\n create annotation := ;\n alter annotation := ;\n drop annotation ;\n using ( ) ;\n using ;\n\n\nDescription\n^^^^^^^^^^^\n\nThe command ``alter function`` changes the definition of a function.\nThe command allows changing annotations, the volatility level, and\nother attributes.\n\n\nSubcommands\n^^^^^^^^^^^\n\nThe following subcommands are allowed in the ``alter function`` block\nin addition to the commands common to the ``create function``:\n\n:eql:synopsis:`reset volatility`\n Remove explicitly specified volatility in favor of the volatility\n inferred from the function body.\n\n:eql:synopsis:`rename to `\n Change the name of the function to *newname*.\n\n:eql:synopsis:`alter annotation ;`\n Alter function :eql:synopsis:``.\n See :eql:stmt:`alter annotation` for details.\n\n:eql:synopsis:`drop annotation ;`\n Remove function :eql:synopsis:``.\n See :eql:stmt:`drop annotation` for details.\n\n:eql:synopsis:`reset errmessage;`\n Remove the error message from this abstract constraint.\n The error message specified in the base abstract constraint\n will be used instead.\n\n\nExample\n^^^^^^^\n\n.. code-block:: edgeql\n\n create function mysum(a: int64, b: int64) -> int64 {\n using (\n select a + b\n );\n create annotation title := \"My sum function.\";\n };\n\n alter function mysum(a: int64, b: int64) {\n set volatility := 'Immutable';\n drop annotation title;\n };\n\n alter function mysum(a: int64, b: int64) {\n using (\n select (a + b) * 100\n )\n };\n\n\nDrop function\n-------------\n\n:eql-statement:\n:eql-haswith:\n\n\nRemove a function.\n\n.. eql:synopsis::\n\n [ with [, ...] ]\n drop function ([ ] [, ... ]);\n\n # where is:\n\n [ ] : [ ] [ = ]\n\n\nDescription\n^^^^^^^^^^^\n\nThe command ``drop function`` removes the definition of an existing function.\nThe argument types to the function must be specified, since there\ncan be different functions with the same name.\n\n\nParameters\n^^^^^^^^^^\n\n:eql:synopsis:``\n The name (optionally module-qualified) of an existing function.\n\n:eql:synopsis:``\n The name of an argument used in the function definition.\n\n:eql:synopsis:``\n The mode of an argument: ``set of`` or ``optional`` or ``variadic``.\n\n:eql:synopsis:``\n The data type(s) of the function's arguments\n (optionally module-qualified), if any.\n\n\nExample\n^^^^^^^\n\nRemove the ``mysum`` function:\n\n.. code-block:: edgeql\n\n drop function mysum(a: int64, b: int64);\n\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`Reference > Function calls `\n * - :ref:`Introspection > Functions `\n * - :ref:`Cheatsheets > Functions `\n\n" }, { "path": "docs/reference/datamodel/future.rst", "content": ".. _ref_datamodel_future:\n\n===============\nFuture behavior\n===============\n\n.. api-index:: using future\n\nThis article explains what the ``using future ...;`` statement means in your\nschema.\n\nOur goal is to make |Gel| the best database system in the world, which requires\nus to keep evolving. Usually, we can add new functionality while preserving\nbackward compatibility, but on rare occasions we must implement changes that\nrequire elaborate transitions.\n\nTo handle these cases, we introduce *future* behavior, which lets you try out\nupcoming features before a major release. Sometimes enabling a future is\nnecessary to fix current issues; other times it offers a safe and easy way to\nensure your codebase remains compatible. This approach provides more time to\nadopt a new feature and identify any resulting bugs.\n\nAny time a behavior is available as a ``future,`` all new :ref:`projects\n` enable it by default for empty databases. You can remove\na ``future`` from your schema if absolutely necessary, but doing so is\ndiscouraged. Existing projects are unaffected by default, so you must manually\nadd the ``future`` specification to gain early access.\n\nFlags\n=====\n\n.. api-index:: simple_scoping, warn_old_scoping, nonrecursive_access_policies\n\nAt the moment there are three ``future`` flags available:\n\n- ``simple_scoping``\n\n Introduced in |Gel| 6.0, this flag simplifies the scoping rules for\n path expressions. Read more about it and in great detail in\n :ref:`ref_eql_path_resolution`.\n\n- ``warn_old_scoping``\n\n Introduced in |Gel| 6.0, this flag will emit a warning when a query\n is detected to depend on the old scoping rules. This is an intermediate\n step towards enabling the ``simple_scoping`` flag in existing large\n codebases.\n\n Read more about this flag in :ref:`ref_warn_old_scoping`.\n\n.. _ref_datamodel_access_policies_nonrecursive:\n.. _nonrecursive:\n\n- ``nonrecursive_access_policies``: makes access policies non-recursive.\n\n This flag is no longer used becauae the behavior is enabled\n by default since |EdgeDB| 4. The flag was helpful to ease transition\n from EdgeDB 3.x to 4.x.\n\n Since |EdgeDB| 3.0, access policy restrictions do **not** apply\n to any access policy expression. This means that when reasoning about access\n policies it is no longer necessary to take other policies into account.\n Instead, all data is visible for the purpose of *defining* an access\n policy.\n\n This change was made to simplify reasoning about access policies and\n to allow certain patterns to be expressed efficiently. Since those who have\n access to modifying the schema can remove unwanted access policies, no\n additional security is provided by applying access policies to each\n other's expressions.\n\n\n.. _ref_eql_sdl_future:\n\nDeclaring future flags\n======================\n\nSyntax\n------\n\nDeclare that the current schema enables a particular future behavior.\n\n.. sdl:synopsis::\n\n using future \";\"\n\nDescription\n^^^^^^^^^^^\n\nFuture behavior declaration must be outside any :ref:`module block\n` since this behavior affects the entire database and not\na specific module.\n\nExample\n^^^^^^^\n\n.. code-block:: sdl-invalid\n\n using future simple_scoping;\n\n\n.. _ref_eql_ddl_future:\n\nDDL commands\n============\n\nThis section describes the low-level DDL commands for creating and\ndropping future flags. You typically don't need to use these commands directly,\nbut knowing about them is useful for reviewing migrations.\n\nCreate future\n-------------\n\n:eql-statement:\n\nEnable a particular future behavior for the current schema.\n\n.. eql:synopsis::\n\n create future \";\"\n\n\nThe command ``create future`` enables the specified future behavior for\nthe current branch.\n\nExample\n^^^^^^^\n\n.. code-block:: edgeql\n\n create future simple_scoping;\n\n\nDrop future\n-----------\n\n:eql-statement:\n\nDisable a particular future behavior for the current schema.\n\n.. eql:synopsis::\n\n drop future \";\"\n\nDescription\n^^^^^^^^^^^\n\nThe command ``drop future`` disables a currently active future behavior for the\ncurrent branch. However, this is only possible for versions of |Gel| when the\nbehavior in question is not officially introduced. Once a particular behavior is\nintroduced as the standard behavior in a |Gel| release, it cannot be disabled.\n\nExample\n^^^^^^^\n\n.. code-block:: edgeql\n\n drop future warn_old_scoping;\n" }, { "path": "docs/reference/datamodel/globals.rst", "content": ".. _ref_datamodel_globals:\n\n=======\nGlobals\n=======\n\nSchemas in Gel can contain typed *global variables*. These create a mechanism\nfor specifying session-level context that can be referenced in queries,\naccess policies, triggers, and elsewhere with the ``global`` keyword.\n\nHere's a very common example of a global variable representing the current\nuser ID:\n\n.. code-block:: sdl\n\n global current_user_id: uuid;\n\n.. tabs::\n\n .. code-tab:: edgeql\n\n select User {\n id,\n posts: { title, content }\n }\n filter .id = global current_user_id;\n\n .. code-tab:: python\n\n # In a non-trivial example, `global current_user_id` would\n # be used indirectly in an access policy or some other context.\n await client.with_globals({'user_id': user_id}).qeury('''\n select User {\n id,\n posts: { title, content }\n }\n filter .id = global current_user_id;\n ''')\n\n .. code-tab:: typescript\n\n // In a non-trivial example, `global current_user_id` would\n // be used indirectly in an access policy or some other context.\n await client.withGlobals({user_id}).qeury('''\n select User {\n id,\n posts: { title, content }\n }\n filter .id = global current_user_id;\n ''')\n\n\nSetting global variables\n========================\n\nGlobal variables are set at session level or when initializing a client.\nThe exact API depends on which client library you're using, but the general\nbehavior and principles are the same across all libraries.\n\n.. tabs::\n\n .. code-tab:: typescript\n\n import createClient from 'gel';\n\n const baseClient = createClient();\n\n // returns a new Client instance, that shares the underlying\n // network connection with `baseClient` , but sends the configured\n // globals along with all queries run through it:\n const clientWithGlobals = baseClient.withGlobals({\n current_user_id: '2141a5b4-5634-4ccc-b835-437863534c51',\n });\n\n const result = await clientWithGlobals.query(\n `select global current_user_id;`\n );\n\n .. code-tab:: python\n\n from gel import create_client\n\n base_client = create_client()\n\n # returns a new Client instance, that shares the underlying\n # network connection with `base_client` , but sends the configured\n # globals along with all queries run through it:\n client = base_client.with_globals({\n 'current_user_id': '580cc652-8ab8-4a20-8db9-4c79a4b1fd81'\n })\n\n result = client.query(\"\"\"\n select global current_user_id;\n \"\"\")\n\n .. code-tab:: go\n\n package main\n\n import (\n \"context\"\n \"fmt\"\n \"log\"\n\n \"github.com/geldata/gel-go\"\n )\n\n func main() {\n ctx := context.Background()\n client, err := gel.CreateClient(ctx, gel.Options{})\n if err != nil {\n log.Fatal(err)\n }\n defer client.Close()\n\n id, err := gel.ParseUUID(\"2141a5b4-5634-4ccc-b835-437863534c51\")\n if err != nil {\n log.Fatal(err)\n }\n\n var result gel.UUID\n err = client.\n WithGlobals(map[string]interface{}{\"current_user\": id}).\n QuerySingle(ctx, \"SELECT global current_user;\", &result)\n if err != nil {\n log.Fatal(err)\n }\n\n fmt.Println(result)\n }\n\n .. code-tab:: rust\n\n use uuid::Uuid;\n\n let client = gel_tokio::create_client().await.expect(\"Client init\");\n\n let client_with_globals = client.with_globals_fn(|c| {\n c.set(\n \"current_user_id\",\n Value::Uuid(\n Uuid::parse_str(\"2141a5b4-5634-4ccc-b835-437863534c51\")\n .expect(\"Uuid should have parsed\"),\n ),\n )\n });\n let val: Uuid = client_with_globals\n .query_required_single(\"select global current_user_id;\", &())\n .await\n .expect(\"Returning value\");\n println!(\"Result: {val}\");\n\n .. code-tab:: edgeql\n\n set global current_user_id :=\n '2141a5b4-5634-4ccc-b835-437863534c51';\n\n\nCardinality\n===========\n\nA global variable can be declared with one of two cardinalities:\n\n- ``single`` (the default): At most one value.\n- ``multi``: A set of values. Only valid for computed global variables.\n\nIn addition, a global can be marked ``required`` or ``optional`` (the default).\nIf marked ``required``, a default value must be provided.\n\n\nComputed globals\n================\n\n.. api-index:: global, :=\n\nGlobal variables can also be computed. The value of computed globals is\ndynamically computed when they are referenced in queries.\n\n.. code-block:: sdl\n\n required global now := datetime_of_transaction();\n\nThe provided expression will be computed at the start of each query in which\nthe global is referenced. There's no need to provide an explicit type; the type\nis inferred from the computed expression.\n\nComputed globals can also be object-typed and have ``multi`` cardinality.\nFor example:\n\n.. code-block:: sdl\n\n global current_user_id: uuid;\n\n # object-typed global\n global current_user := (\n select User filter .id = global current_user_id\n );\n\n # multi global\n global current_user_friends := (global current_user).friends;\n\n\nReferencing globals\n===================\n\n.. api-index:: global\n\nUnlike query parameters, globals can be referenced *inside your schema\ndeclarations*:\n\n.. code-block:: sdl\n\n type User {\n name: str;\n is_self := (.id = global current_user_id)\n };\n\nThis is particularly useful when declaring :ref:`access policies\n`:\n\n.. code-block:: sdl\n\n type Person {\n required name: str;\n\n access policy my_policy allow all\n using (.id = global current_user_id);\n }\n\nRefer to :ref:`Access Policies ` for complete\ndocumentation.\n\n.. _ref_eql_sdl_globals:\n.. _ref_eql_sdl_globals_syntax:\n\nDeclaring globals\n=================\n\n.. api-index:: required, optional, single, multi, global, :=, :, default\n\nThis section describes the syntax to declare a global variable in your schema.\n\nSyntax\n------\n\nDefine a new global variable in SDL, corresponding to the more explicit DDL\ncommands described later:\n\n.. sdl:synopsis::\n\n # Global variable declaration:\n [{required | optional}] [single]\n global : \n [ \"{\"\n [ default := ; ]\n [ ]\n ...\n \"}\" ]\n\n # Computed global variable declaration:\n [{required | optional}] [{single | multi}]\n global := ;\n\n\nDescription\n^^^^^^^^^^^\n\nThere are two different forms of ``global`` declarations, as shown in the\nsyntax synopsis above:\n\n1. A *settable* global (defined with ``: ``) which can be changed using\n a session-level :ref:`set ` command.\n\n2. A *computed* global (defined with ``:= ``), which cannot be\n directly set but instead derives its value from the provided expression.\n\nThe following options are available:\n\n:eql:synopsis:`required`\n If specified, the global variable is considered *required*. It is an\n error for this variable to have an empty value. If a global variable is\n declared *required*, it must also declare a *default* value.\n\n:eql:synopsis:`optional`\n The global variable is considered *optional*, i.e. it is possible for the\n variable to have an empty value. (This is the default.)\n\n:eql:synopsis:`multi`\n Specifies that the global variable may have a set of values. Only\n *computed* global variables can have this qualifier.\n\n:eql:synopsis:`single`\n Specifies that the global variable must have at most a *single* value. It\n is assumed that a global variable is ``single`` if neither ``multi`` nor\n ``single`` is specified. All non-computed global variables must be *single*.\n\n:eql:synopsis:``\n The name of the global variable. It can be fully-qualified with the module\n name, or it is assumed to belong to the module in which it appears.\n\n:eql:synopsis:``\n The type must be a valid :ref:`type expression ` denoting a\n non-abstract scalar or a container type.\n\n:eql:synopsis:` := `\n Defines a *computed* global variable. The provided expression must be a\n :ref:`Stable ` EdgeQL expression. It can refer\n to other global variables. The type of a *computed* global variable is\n not limited to scalar and container types; it can also be an object type.\n\nThe valid SDL sub-declarations are:\n\n:eql:synopsis:`default := `\n Specifies the default value for the global variable as an EdgeQL\n expression. The default value is used in a session if the value was not\n explicitly specified by the client, or was reset with the :ref:`reset\n ` command.\n\n:sdl:synopsis:``\n Set global variable :ref:`annotation `\n to a given *value*.\n\n\nExamples\n--------\n\nDeclare a new global variable:\n\n.. code-block:: sdl\n\n global current_user_id: uuid;\n global current_user := (\n select User filter .id = global current_user_id\n );\n\nSet the global variable to a specific value using :ref:`session-level commands\n`:\n\n.. code-block:: edgeql\n\n set global current_user_id :=\n '00ea8eaa-02f9-11ed-a676-6bd11cc6c557';\n\nUse the computed global variable that is based on the value that was just set:\n\n.. code-block:: edgeql\n\n select global current_user { name };\n\n:ref:`Reset ` the global variable to\nits default value:\n\n.. code-block:: edgeql\n\n reset global user_id;\n\n\n.. _ref_eql_ddl_globals:\n\n\nDDL commands\n============\n\nThis section describes the low-level DDL commands for creating, altering, and\ndropping globals. You typically don't need to use these commands directly, but\nknowing about them is useful for reviewing migrations.\n\n\nCreate global\n-------------\n\n:eql-statement:\n:eql-haswith:\n\nDeclare a new global variable using DDL.\n\n.. eql:synopsis::\n\n [ with [, ...] ]\n create [{required | optional}] [single]\n global : \n [ \"{\" ; [...] \"}\" ] ;\n\n # Computed global variable form:\n\n [ with [, ...] ]\n create [{required | optional}] [{single | multi}]\n global := ;\n\n # where is one of\n\n set default := \n create annotation := \n\nDescription\n^^^^^^^^^^^\n\nAs with SDL, there are two different forms of ``global`` declaration:\n\n- A global variable that can be :ref:`set `\n in a session.\n- A *computed* global that is derived from an expression (and so cannot be\n directly set in a session).\n\nThe subcommands mirror those in SDL:\n\n:eql:synopsis:`set default := `\n Specifies the default value for the global variable as an EdgeQL\n expression. The default value is used by the session if the value was not\n explicitly specified or was reset with the :ref:`reset\n ` command.\n\n:eql:synopsis:`create annotation := `\n Assign an annotation to the global variable. See :eql:stmt:`create annotation`\n for details.\n\n\nExamples\n^^^^^^^^\n\nDefine a new global property ``current_user_id``:\n\n.. code-block:: edgeql\n\n create global current_user_id: uuid;\n\nDefine a new *computed* global property ``current_user`` based on the\npreviously defined ``current_user_id``:\n\n.. code-block:: edgeql\n\n create global current_user := (\n select User filter .id = global current_user_id\n );\n\n\nAlter global\n------------\n\n:eql-statement:\n:eql-haswith:\n\nChange the definition of a global variable.\n\n.. eql:synopsis::\n\n [ with [, ...] ]\n alter global \n [ \"{\" ; [...] \"}\" ] ;\n\n # where is one of\n\n set default := \n reset default\n rename to \n set required\n set optional\n reset optionalily\n set single\n set multi\n reset cardinality\n set type reset to default\n using ()\n create annotation := \n alter annotation := \n drop annotation \n\nDescription\n^^^^^^^^^^^\n\nThe command :eql:synopsis:`alter global` changes the definition of a global\nvariable. It can modify default values, rename the global, or change other\nattributes like optionality, cardinality, computed expressions, etc.\n\nExamples\n^^^^^^^^\n\nSet the ``description`` annotation of global variable ``current_user``:\n\n.. code-block:: edgeql\n\n alter global current_user\n create annotation description :=\n 'Current User as specified by the global ID';\n\nMake the ``current_user_id`` global variable ``required``:\n\n.. code-block:: edgeql\n\n alter global current_user_id {\n set required;\n # A required global variable MUST have a default value.\n set default := '00ea8eaa-02f9-11ed-a676-6bd11cc6c557';\n }\n\n\nDrop global\n-----------\n\n:eql-statement:\n:eql-haswith:\n\nRemove a global variable from the schema.\n\n.. eql:synopsis::\n\n [ with [, ...] ]\n drop global ;\n\nDescription\n^^^^^^^^^^^\n\nThe command :eql:synopsis:`drop global` removes the specified global variable\nfrom the schema.\n\nExample\n^^^^^^^\n\nRemove the ``current_user`` global variable:\n\n.. code-block:: edgeql\n\n drop global current_user;\n" }, { "path": "docs/reference/datamodel/index.rst", "content": ".. versioned-section::\n\n.. _ref_datamodel_index:\n\n======\nSchema\n======\n\n.. toctree::\n :maxdepth: 3\n :hidden:\n\n objects\n properties\n links\n computeds\n primitives\n indexes\n constraints\n inheritance\n aliases\n globals\n access_policies\n permissions\n functions\n triggers\n mutation_rewrites\n linkprops\n modules\n migrations\n branches\n extensions\n annotations\n future\n comparison\n introspection/index\n\n\n|Gel| schema is a high-level description of your application's data model.\nIn the schema, you define your types, links, access policies, functions,\ntriggers, constraints, indexes, and more.\n\nGel schema is strictly typed and is high-level enough to be mapped directly\nto mainstream programming languages and back.\n\n\n.. _ref_eql_sdl:\n\nSchema Definition Language\n==========================\n\nMigrations are sequences of *data definition language* (DDL) commands.\nDDL is a low-level language that tells the database exactly how to change\nthe schema. You typically won't need to write any DDL by hand; the Gel server\nwill generate it for you.\n\nFor a full guide on migrations, refer to the :ref:`Creating and applying\nmigrations ` guide or the\n:ref:`migrations reference ` section.\n\n\nExample:\n\n.. code-block:: sdl\n\n # dbschema/default.gel\n\n type Movie {\n required title: str;\n required director: Person;\n }\n\n type Person {\n required name: str;\n }\n\n.. important::\n\n Syntax highlighter packages/extensions for |.gel| files are available for\n `Visual Studio Code `_,\n `Sublime Text `_,\n `Atom `_, and `Vim `_.\n\nMigrations and DDL\n==================\n\nGel's baked-in migration system lets you painlessly evolve your schema over\ntime. Just update the contents of your |.gel| file(s) and use the |Gel| CLI\nto *create* and *apply* migrations.\n\n.. code-block:: bash\n\n $ gel migration create\n Created dbschema/migrations/00001.edgeql\n $ gel migrate\n Applied dbschema/migrations/00001.edgeql\n\nMigrations are sequences of *data definition language* (DDL) commands.\nDDL is a low level language that tells the database how exactly to change\nthe schema. Don't worry, you won't need to write any DDL directly, the Gel\nserver will generate it for you.\n\nFor a full guide on migrations, refer to the :ref:`Creating and applying\nmigrations ` guide or the\n:ref:`migrations reference ` section.\n\n\n.. _ref_datamodel_terminology:\n.. _ref_datamodel_instances:\n\nInstances, branches, and modules\n================================\n\nGel is like a stack of containers:\n\n* The *instance* is the running Gel process. Every instance has one or\n more |branches|. Instances can be created, started, stopped, and\n destroyed locally with :ref:`gel project `\n or low-level :ref:`gel instance ` commands.\n\n* A *branch* is where your schema and data live. Branches map to PostgreSQL\n databases. Like instances, branches can be conveniently created, removed,\n and switched with the :ref:`gel branch ` commands.\n Read more about branches in the\n :ref:`branches reference `.\n\n* A *module* is a collection of types, functions, and other definitions.\n The default module is called ``default``. Modules are used to organize\n your schema logically. Read more about modules in the\n :ref:`modules reference `.\n" }, { "path": "docs/reference/datamodel/indexes.rst", "content": ".. _ref_datamodel_indexes:\n\n=======\nIndexes\n=======\n\n.. index:: performance, postgres query planner\n\nAn index is a data structure used internally to speed up filtering, ordering,\nand grouping operations in |Gel|. Indexes help accomplish this in two key ways:\n\n- They are pre-sorted, which saves time on costly sort operations on rows.\n- They can be used by the query planner to filter out irrelevant rows.\n\n.. note::\n\n The Postgres query planner decides when to use indexes for a query. In some\n casesā€”e.g. when tables are smallā€”it may be faster to scan the whole table\n rather than use an index. In such scenarios, the index might be ignored.\n\n For more information on how the planner decides this, see\n `the Postgres query planner documentation\n `_.\n\n\nTradeoffs\n=========\n\nWhile improving query performance, indexes also increase disk and memory usage\nand can slow down insertions and updates. Creating too many indexes may be\ndetrimental; only index properties you often filter, order, or group by.\n\n.. important::\n **Foreign and primary keys**\n\n In SQL databases, indexes are commonly used to index *primary keys* and\n *foreign keys*. Gel's analog to a SQL primary key is the ``id`` field\n automatically created for each object, while a link in Gel is the analog\n to a SQL foreign key. Both of these are automatically indexed.\n\n Moreover, any property with an :eql:constraint:`exclusive` constraint\n is also automatically indexed.\n\n\nIndex on a property\n===================\n\nMost commonly, indexes are declared within object type declarations and\nreference a particular property. The index can be used to speed up queries\nthat reference that property in a filter, order by, or group by clause:\n\n.. code-block:: sdl\n\n type User {\n required name: str;\n index on (.name);\n }\n\nBy indexing on ``User.name``, the query planner will have access to that index\nwhen planning queries using the ``name`` property. This may result in better\nperformance as the database can look up a name in the index instead of scanning\nthrough all ``User`` objects sequentiallyā€”though ultimately it's up to the\nPostgres query planner whether to use the index.\n\nTo see if an index helps, compare query plans by adding\n:ref:`analyze ` to your queries.\n\n.. note::\n\n Even if your database is small now, you may benefit from an index as it grows.\n\n\nIndex on an expression\n======================\n\nIndexes may be defined using an arbitrary *singleton* expression that\nreferences multiple properties of the enclosing object type.\n\n.. important::\n A singleton expression is an expression that's guaranteed to return\n *at most one* element. As such, you can't index on a ``multi`` property.\n\nExample:\n\n.. code-block:: sdl\n\n type User {\n required first_name: str;\n required last_name: str;\n index on (str_lower(.first_name + ' ' + .last_name));\n }\n\n\nIndex on multiple properties\n============================\n\nA *composite index* references multiple properties. This can speed up queries\nthat filter, order, or group on multiple properties at once.\n\n.. note::\n\n An index on multiple properties may also be used in queries where only a\n single property in the index is referenced. In many traditional database\n systems, placing the most frequently used columns first in the composite\n index can improve the likelihood of its use.\n\n Read `the Postgres documentation on multicolumn indexes\n `_ to learn\n more about how the query planner uses these indexes.\n\nIn |Gel|, a composite index is created by indexing on a ``tuple`` of properties:\n\n.. code-block:: sdl\n\n type User {\n required name: str;\n required email: str;\n index on ((.name, .email));\n }\n\n\nIndex on a link property\n========================\n\nLink properties can also be indexed. The special placeholder\n``__subject__`` refers to the source object in a link property expression:\n\n.. code-block:: sdl\n\n abstract link friendship {\n strength: float64;\n index on (__subject__@strength);\n }\n\n type User {\n multi friends: User {\n extending friendship;\n };\n }\n\nExclude objects from an index\n=============================\n\nWhen specifying an index, you can provide an optional ``except`` clause to exclude objects from the index. This is known as creating a *partial index*. Partial indexes are particularly useful in scenarios where you frequently query a subset of data that meets certain criteria, while consistently excluding other data. For example, if you often filter on a property but always exclude objects with a specific value for another property, a partial index can optimize these queries by indexing only the relevant subset of data, thus improving query performance and reducing index size.\n\n.. code-block:: sdl\n\n type User {\n required name: str;\n required email: str;\n archived_at: datetime;\n\n index on (.name) except (exists .archived_at);\n }\n\n\nSpecify a Postgres index type\n=============================\n\n.. api-index:: pg::hash, pg::btree, pg::gin, pg::gist, pg::spgist, pg::brin\n\n.. versionadded:: 3.0\n\nGel exposes Postgres index types that can be used directly in schemas via\nthe ``pg`` module:\n\n- ``pg::hash`` : Index based on a 32-bit hash of the value\n- ``pg::btree`` : B-tree index (can help with sorted data retrieval)\n- ``pg::gin`` : Inverted index for multi-element data (arrays, JSON)\n- ``pg::gist`` : Generalized Search Tree for range and geometric searches\n- ``pg::spgist`` : Space-partitioned GiST\n- ``pg::brin`` : Block Range INdex\n\nExample:\n\n.. code-block:: sdl\n\n type User {\n required name: str;\n index pg::spgist on (.name);\n }\n\n\n.. _ref_datamodel_indexes_concurrent:\n\nConcurrent index building\n=========================\n\nWhen creating an index, the object type will be locked for writes. This means\nthat until the index is created, all ``insert``, ``update`` and ``delete``\nqueries will be put on hold.\nOn types containing many objects, this can span minutes or even hours.\n\nInstead, index building can be deferred from migration application to a later\ntime. To do this, set ``build_concurrently`` index property to ``true``:\n\n.. code-block:: sdl\n\n type User {\n name: str;\n index on (.name) {\n build_concurrently := true;\n };\n }\n\nWhen this schema in applied to an instance, the index will be created, but it\nwill not yet be active. The migration will not attempt to read any objects to\nbuild the index.\n\nAs the last step of :gelcmd:`migration apply` (and :gelcmd:`migrate`), index\nwill actually be built. During this time, the object type will not be locked\nfor reads or writes.\n\nThis means that migration will lock for significantly less time and allow index\nthe be created while new writes are applied to the database.\n\nTo apply migrations, but not build indexes at all, use\n:gelcmd:`migration apply --no-index-build` flag.\nThis allows index building to be triggered at a later time,\nby using :gelcmd:`migration apply` again.\n\nUntil the index is created, it will not be used to speed up queries.\n\nFor tradeoffs of concurrent index building, refer to\n`PostgreSQL documentation `_.\n\n\nAnnotate an index\n=================\n\nIndexes can include annotations:\n\n.. code-block:: sdl\n\n type User {\n name: str;\n index on (.name) {\n annotation description := 'Indexing all users by name.';\n };\n }\n\n\n.. _ref_eql_sdl_indexes:\n\nDeclaring indexes\n=================\n\n.. api-index:: index on, except\n\nThis section describes the syntax to use indexes in your schema.\n\nSyntax\n------\n\n.. sdl:synopsis::\n\n index on ( )\n [ except ( ) ]\n [ \"{\" \"}\" ] ;\n\n.. rubric:: Description\n\n- :sdl:synopsis:`on ( )`\n\n The expression to index. It must be :ref:`Immutable `\n but may refer to the indexed object's properties/links. The expression itself\n must be parenthesized.\n\n- :eql:synopsis:`except ( )`\n\n An optional condition. If ```` evaluates to ``true``, the object\n is omitted from the index; if ``false`` or empty, it is included.\n\n- :sdl:synopsis:``\n\n Allows setting index :ref:`annotation ` to a given\n value.\n\n- :sdl:synopsis:`build_concurrently := `\n\n Allows index to be built\n :ref:`after migration is applied `\n to the instance.\n\n.. _ref_eql_ddl_indexes:\n\nDDL commands\n============\n\nThis section describes the low-level DDL commands for creating, altering, and\ndropping indexes. You typically don't need to use these commands directly, but\nknowing about them is useful for reviewing migrations.\n\n\nCreate index\n------------\n\n:eql-statement:\n\n.. eql:synopsis::\n\n create index on ( )\n [ except ( ) ]\n [ \"{\" ; [...] \"}\" ] ;\n\n # where is one of\n\n create annotation := \n\nCreates a new index for a given object type or link using *index-expr*.\n\n- Most parameters/options match those in\n :ref:`Declaring indexes `.\n\n- Allowed subcommand:\n\n :eql:synopsis:`create annotation := `\n Assign an annotation to this index.\n See :eql:stmt:`create annotation` for details.\n\n- :eql:synopsis:`set build_concurrently := `\n\n Allows index to be built\n :ref:`after migration is applied `\n to the instance.\n\nExample:\n\n.. code-block:: edgeql\n\n create type User {\n create property name: str {\n set default := '';\n };\n\n create index on (.name);\n };\n\n\nAlter index\n-----------\n\n:eql-statement:\n\nAlter the definition of an index.\n\n.. eql:synopsis::\n\n alter index on ( ) [ except ( ) ]\n [ \"{\" ; [...] \"}\" ] ;\n\n # where is one of\n\n create annotation := \n alter annotation := \n drop annotation \n\nThe command ``alter index`` is used to change the :ref:`annotations\n` of an index. The *index-expr* is used to\nidentify the index to be altered.\n\n:sdl:synopsis:`on ( )`\n The specific expression for which the index is made. Note also\n that ```` itself has to be parenthesized.\n\nThe following subcommands are allowed in the ``alter index`` block:\n\n:eql:synopsis:`create annotation := `\n Set index :eql:synopsis:`` to\n :eql:synopsis:``.\n See :eql:stmt:`create annotation` for details.\n\n:eql:synopsis:`alter annotation ;`\n Alter index :eql:synopsis:``.\n See :eql:stmt:`alter annotation` for details.\n\n:eql:synopsis:`drop annotation ;`\n Remove constraint :eql:synopsis:``.\n See :eql:stmt:`drop annotation` for details.\n\n\nExample:\n\n.. code-block:: edgeql\n\n alter type User {\n alter index on (.name) {\n create annotation title := 'User name index';\n };\n };\n\n\nDrop index\n----------\n\n:eql-statement:\n\nRemove an index from a given schema item.\n\n.. eql:synopsis::\n\n drop index on ( ) [ except ( ) ] ;\n\nRemoves an index from a schema item.\n\n- :sdl:synopsis:`on ( )` identifies the indexed expression.\n\nThis statement can only be used as a subdefinition in another DDL statement.\n\nExample:\n\n.. code-block:: edgeql\n\n alter type User {\n drop index on (.name);\n };\n\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n - :ref:`Introspection > Indexes `\n" }, { "path": "docs/reference/datamodel/inheritance.rst", "content": ".. _ref_datamodel_inheritance:\n\n===========\nInheritance\n===========\n\n.. index:: extending, extends, subtype, supertype, parent type, child type\n\nInheritance is a crucial aspect of schema modeling in Gel. Schema items can\n*extend* one or more parent types. When extending, the child (subclass)\ninherits the definition of its parents (superclass).\n\nYou can declare ``abstract`` object types, properties, links, constraints, and\nannotations.\n\n- :ref:`Objects `\n- :ref:`Properties `\n- :ref:`Links `\n- :ref:`Constraints `\n- :ref:`Annotations `\n\n.. _ref_datamodel_inheritance_objects:\n\nObject types\n------------\n\n.. api-index:: abstract type, extending\n\nObject types can *extend* other object types. The extending type (AKA the\n*subtype*) inherits all links, properties, indexes, constraints, etc. from its\n*supertypes*.\n\n.. code-block:: sdl\n\n abstract type Animal {\n species: str;\n }\n\n type Dog extending Animal {\n breed: str;\n }\n\nBoth abstract and concrete object types can be extended. Whether to make a\ntype abstract or concrete is a fairly simple decision: if you need to be\nable to insert objects of the type, make it a concrete type. If objects of\nthe type should never be inserted and it exists only to be extended, make it\nan abstract one. In the schema below the ``Animal`` type is now concrete\nand can be inserted, which was not the case in the example above. The new\n``CanBark`` type however is abstract and thus the database will not have\nany individual ``CanBark`` objects.\n\n.. code-block:: sdl\n\n abstract type CanBark {\n required bark_sound: str;\n }\n\n type Animal {\n species: str;\n }\n\n type Dog extending Animal, CanBark {\n breed: str;\n }\n\n\nFor details on querying polymorphic data, see :ref:`EdgeQL > Select >\nPolymorphic queries `. When using the SQL adapter,\nsee :ref:`SQL adapter ` for information about using ``ONLY``\nto query parent tables without including child objects.\n\n.. _ref_datamodel_inheritance_multiple:\n\nMultiple Inheritance\n^^^^^^^^^^^^^^^^^^^^\n\nObject types can :ref:`extend more\nthan one type ` ā€” that's called\n*multiple inheritance*. This mechanism allows building complex object\ntypes out of combinations of more basic types.\n\n.. code-block:: sdl\n\n abstract type HasName {\n first_name: str;\n last_name: str;\n }\n\n abstract type HasEmail {\n email: str;\n }\n\n type Person extending HasName, HasEmail {\n profession: str;\n }\n\n\n.. _ref_datamodel_overloading:\n\nOverloading\n^^^^^^^^^^^\n\n.. api-index:: overloaded\n\nAn object type can overload an inherited property or link. All overloaded\ndeclarations must be prefixed with the ``overloaded`` prefix to avoid\nunintentional overloads.\n\n.. code-block:: sdl\n\n abstract type Person {\n name: str;\n multi friends: Person;\n }\n\n type Student extending Person {\n overloaded name: str {\n constraint exclusive;\n }\n overloaded multi friends: Student;\n }\n\n\nOverloaded fields cannot *generalize* the associated type; it can only make it\n*more specific* by setting the type to a subtype of the original or adding\nadditional constraints.\n\n.. _ref_datamodel_inheritance_props:\n\nProperties\n----------\n\n.. api-index:: abstract property, readonly\n\nProperties can be *concrete* (the default) or *abstract*. Abstract properties\nare declared independent of a source or target, can contain :ref:`annotations\n`, and can be marked as ``readonly``.\n\n.. code-block:: sdl\n\n abstract property title_prop {\n annotation title := 'A title.';\n readonly := false;\n }\n\n.. _ref_datamodel_inheritance_links:\n\nLinks\n-----\n\n.. api-index:: abstract link\n\nIt's possible to define ``abstract`` links that aren't tied to a particular\n*source* or *target*. Abstract links can be marked as readonly and contain\nannotations, property declarations, constraints, and indexes.\n\n.. code-block:: sdl\n\n abstract link link_with_strength {\n strength: float64;\n index on (__subject__@strength);\n }\n\n type Person {\n multi friends: Person {\n extending link_with_strength;\n };\n }\n\n.. _ref_datamodel_inheritance_constraints:\n\nConstraints\n-----------\n\n.. api-index:: abstract constraint, using, errmessage\n\nUse ``abstract`` to declare reusable, user-defined constraint types.\n\n.. code-block:: sdl\n\n abstract constraint in_range(min: anyreal, max: anyreal) {\n errmessage :=\n 'Value must be in range [{min}, {max}].';\n using (min <= __subject__ and __subject__ < max);\n }\n\n type Player {\n points: int64 {\n constraint in_range(0, 100);\n }\n }\n\n.. _ref_datamodel_inheritance_annotations:\n\nAnnotations\n-----------\n\n.. api-index:: abstract annotation, inheritable\n\nEdgeQL supports three annotation types by default: ``title``, ``description``,\nand ``deprecated``. Use ``abstract annotation`` to declare custom user-defined\nannotation types.\n\n.. code-block:: sdl\n\n abstract annotation admin_note;\n\n type Status {\n annotation admin_note := 'system-critical';\n # more properties\n }\n\nBy default, annotations defined on abstract types, properties, and links will\nnot be inherited by their subtypes. To override this behavior, use the\n``inheritable`` modifier.\n\n.. code-block:: sdl\n\n abstract inheritable annotation admin_note;\n\n" }, { "path": "docs/reference/datamodel/introspection/casts.rst", "content": ".. _ref_datamodel_introspection_casts:\n\n=====\nCasts\n=====\n\nThis section describes introspection of Gel :eql:op:`type casts\n`. Features like whether the casts are implicit can be\ndiscovered by introspecting ``schema::Cast``.\n\nIntrospection of the ``schema::Cast``:\n\n.. code-block:: edgeql-repl\n\n db> with module schema\n ... select ObjectType {\n ... name,\n ... links: {\n ... name,\n ... },\n ... properties: {\n ... name,\n ... }\n ... }\n ... filter .name = 'schema::Cast';\n {\n Object {\n name: 'schema::Cast',\n links: {\n Object { name: '__type__' },\n Object { name: 'from_type' },\n Object { name: 'to_type' }\n },\n properties: {\n Object { name: 'allow_assignment' },\n Object { name: 'allow_implicit' },\n Object { name: 'id' },\n Object { name: 'name' }\n }\n }\n }\n\nIntrospection of the possible casts from ``std::int64`` to other\ntypes:\n\n.. code-block:: edgeql-repl\n\n db> with module schema\n ... select Cast {\n ... allow_assignment,\n ... allow_implicit,\n ... to_type: { name },\n ... }\n ... filter .from_type.name = 'std::int64'\n ... order by .to_type.name;\n {\n Object {\n allow_assignment: false,\n allow_implicit: true,\n to_type: Object { name: 'std::bigint' }\n },\n Object {\n allow_assignment: false,\n allow_implicit: true,\n to_type: Object { name: 'std::decimal' }\n },\n Object {\n allow_assignment: true,\n allow_implicit: false,\n to_type: Object { name: 'std::float32' }\n },\n Object {\n allow_assignment: false,\n allow_implicit: true,\n to_type: Object { name: 'std::float64' }\n },\n Object {\n allow_assignment: true,\n allow_implicit: false,\n to_type: Object { name: 'std::int16' }\n },\n Object {\n allow_assignment: true,\n allow_implicit: false,\n to_type: Object { name: 'std::int32' }\n },\n Object {\n allow_assignment: false,\n allow_implicit: false,\n to_type: Object { name: 'std::json' }\n },\n Object {\n allow_assignment: false,\n allow_implicit: false,\n to_type: Object { name: 'std::str' }\n }\n }\n\nThe ``allow_implicit`` property tells whether this is an *implicit cast*\nin all contexts (such as when determining the type of a set of mixed\nliterals or resolving the argument types of functions or operators if\nthere's no exact match). For example, a literal ``1`` is an\n:eql:type:`int64` and it is implicitly cast into a :eql:type:`bigint`\nor :eql:type:`float64` if it is added to a set containing either one\nof those types:\n\n.. code-block:: edgeql-repl\n\n db> select {1, 2n};\n {1n, 2n}\n db> select {1, 2.0};\n {1.0, 2.0}\n\nWhat happens if there's no implicit cast between a couple of scalars\nin this type of example? Gel checks whether there's a scalar type\nsuch that all of the set elements can be implicitly cast into that:\n\n.. code-block:: edgeql-repl\n\n db> select introspect (typeof {1, 2}).name;\n {'std::float64'}\n\nThe scalar types :eql:type:`int64` and :eql:type:`float32` cannot be\nimplicitly cast into each other, but they both can be implicitly cast\ninto :eql:type:`float64`.\n\nThe ``allow_assignment`` property tells whether this is an implicit\ncast during assignment if a more general *implicit cast* is not\nallowed. For example, consider the following type:\n\n.. code-block:: sdl\n\n type Example {\n property p_int16: int16;\n property p_float32: float32;\n property p_json: json;\n }\n\n.. code-block:: edgeql-repl\n\n db> insert Example {\n ... p_int16 := 1,\n ... p_float32 := 2\n ... };\n {Object { id: '...' }}\n db> insert Example {\n ... p_json := 3 # assignment cast to json not allowed\n ... };\n InvalidPropertyTargetError: invalid target for property\n 'p_json' of object type 'default::Example': 'std::int64'\n (expecting 'std::json')\n" }, { "path": "docs/reference/datamodel/introspection/colltypes.rst", "content": ".. _ref_datamodel_introspection_collection_types:\n\n================\nCollection types\n================\n\nThis section describes introspection of :ref:`collection types\n`.\n\n\nArray\n-----\n\nIntrospection of the ``schema::Array``:\n\n.. code-block:: edgeql-repl\n\n db> with module schema\n ... select ObjectType {\n ... name,\n ... links: {\n ... name,\n ... },\n ... properties: {\n ... name,\n ... }\n ... }\n ... filter .name = 'schema::Array';\n {\n Object {\n name: 'schema::Array',\n links: {\n Object { name: '__type__' },\n Object { name: 'element_type' }\n },\n properties: {\n Object { name: 'id' },\n Object { name: 'name' }\n }\n }\n }\n\nFor a type with an :eql:type:`array` property, consider the following:\n\n.. code-block:: sdl\n\n type User {\n required property name: str;\n property favorites: array;\n }\n\nIntrospection of the ``User`` with emphasis on properties:\n\n.. code-block:: edgeql-repl\n\n db> with module schema\n ... select ObjectType {\n ... name,\n ... properties: {\n ... name,\n ... target: {\n ... name,\n ... [is Array].element_type: { name },\n ... },\n ... },\n ... }\n ... filter .name = 'default::User';\n {\n Object {\n name: 'default::User',\n properties: {\n Object {\n name: 'favorites',\n target: Object {\n name: 'array',\n element_type: Object { name: 'std::str' }\n }\n },\n ...\n }\n }\n }\n\n\nTuple\n-----\n\nIntrospection of the ``schema::Tuple``:\n\n.. code-block:: edgeql-repl\n\n db> with module schema\n ... select ObjectType {\n ... name,\n ... links: {\n ... name,\n ... },\n ... properties: {\n ... name,\n ... }\n ... }\n ... filter .name = 'schema::Tuple';\n {\n Object {\n name: 'schema::Tuple',\n links: {\n Object { name: '__type__' },\n Object { name: 'element_types' }\n },\n properties: {\n Object { name: 'id' },\n Object { name: 'name' }\n }\n }\n }\n\nFor example, below is an introspection of the return type of\nthe :eql:func:`sys::get_version` function:\n\n.. code-block:: edgeql-repl\n\n db> with module schema\n ... select `Function` {\n ... return_type[is Tuple]: {\n ... element_types: {\n ... name,\n ... type: { name }\n ... } order by .num\n ... }\n ... }\n ... filter .name = 'sys::get_version';\n {\n Object {\n return_type: Object {\n element_types: {\n Object {\n name: 'major',\n type: Object {\n name: 'std::int64'\n }\n },\n Object {\n name: 'minor',\n type: Object {\n name: 'std::int64'\n }\n },\n Object {\n name: 'stage',\n type: Object {\n name: 'sys::VersionStage'\n }\n },\n Object {\n name: 'stage_no',\n type: Object {\n name: 'std::int64'\n }\n },\n Object {\n name: 'local',\n type: Object { name: 'array' }\n }\n }\n }\n }\n }\n" }, { "path": "docs/reference/datamodel/introspection/constraints.rst", "content": ".. _ref_datamodel_introspection_constraints:\n\n===========\nConstraints\n===========\n\nThis section describes introspection of :ref:`constraints\n`.\n\nIntrospection of the ``schema::Constraint``:\n\n.. code-block:: edgeql-repl\n\n db> with module schema\n ... select ObjectType {\n ... name,\n ... links: {\n ... name,\n ... },\n ... properties: {\n ... name,\n ... }\n ... }\n ... filter .name = 'schema::Constraint';\n {\n Object {\n name: 'schema::Constraint',\n links: {\n Object { name: '__type__' },\n Object { name: 'args' },\n Object { name: 'annotations' },\n Object { name: 'bases' },\n Object { name: 'ancestors' },\n Object { name: 'params' },\n Object { name: 'return_type' },\n Object { name: 'subject' }\n },\n properties: {\n Object { name: 'errmessage' },\n Object { name: 'expr' },\n Object { name: 'finalexpr' },\n Object { name: 'id' },\n Object { name: 'abstract' },\n Object { name: 'name' },\n Object { name: 'return_typemod' },\n Object { name: 'subjectexpr' }\n }\n }\n }\n\nConsider the following schema:\n\n.. code-block:: sdl\n\n scalar type maxex_100 extending int64 {\n constraint max_ex_value(100);\n }\n\nIntrospection of the scalar ``maxex_100`` with focus on the constraint:\n\n.. code-block:: edgeql-repl\n\n db> with module schema\n ... select ScalarType {\n ... name,\n ... constraints: {\n ... name,\n ... expr,\n ... annotations: { name, @value },\n ... subject: { name },\n ... params: { name, @value, type: { name } },\n ... return_typemod,\n ... return_type: { name },\n ... errmessage,\n ... },\n ... }\n ... filter .name = 'default::maxex_100';\n {\n Object {\n name: 'default::maxex_100',\n constraints: {\n Object {\n name: 'std::max_ex_value',\n expr: '(__subject__ <= max)',\n annotations: {},\n subject: Object { name: 'default::maxex_100' },\n params: {\n Object {\n name: 'max',\n type: Object { name: 'anytype' },\n @value: '100'\n }\n },\n return_typemod: 'SingletonType',\n return_type: Object { name: 'std::bool' }\n errmessage: '{__subject__} must be less ...',\n }\n }\n }\n }\n\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`Schema > Constraints `\n * - :ref:`SDL > Constraints `\n * - :ref:`DDL > Constraints `\n * - :ref:`Standard Library > Constraints `\n" }, { "path": "docs/reference/datamodel/introspection/functions.rst", "content": ".. _ref_datamodel_introspection_functions:\n\n=========\nFunctions\n=========\n\nThis section describes introspection of :ref:`functions\n`.\n\nIntrospection of the ``schema::Function``:\n\n.. code-block:: edgeql-repl\n\n db> with module schema\n ... select ObjectType {\n ... name,\n ... links: {\n ... name,\n ... },\n ... properties: {\n ... name,\n ... }\n ... }\n ... filter .name = 'schema::Function';\n {\n Object {\n name: 'schema::Function',\n links: {\n Object { name: '__type__' },\n Object { name: 'annotations' },\n Object { name: 'params' },\n Object { name: 'return_type' }\n },\n properties: {\n Object { name: 'id' },\n Object { name: 'name' },\n Object { name: 'return_typemod' }\n }\n }\n }\n\nSince ``params`` are quite important to functions, here's their structure:\n\n.. code-block:: edgeql-repl\n\n db> with module schema\n ... select ObjectType {\n ... name,\n ... links: {\n ... name,\n ... },\n ... properties: {\n ... name,\n ... }\n ... }\n ... filter .name = 'schema::Parameter';\n {\n Object {\n name: 'schema::Parameter',\n links: {\n Object { name: '__type__' },\n Object { name: 'type' }\n },\n properties: {\n Object { name: 'default' },\n Object { name: 'id' },\n Object { name: 'kind' },\n Object { name: 'name' },\n Object { name: 'num' },\n Object { name: 'typemod' }\n }\n }\n }\n\nIntrospection of the built-in :eql:func:`count`:\n\n.. code-block:: edgeql-repl\n\n db> with module schema\n ... select `Function` {\n ... name,\n ... annotations: { name, @value },\n ... params: {\n ... kind,\n ... name,\n ... num,\n ... typemod,\n ... type: { name },\n ... default,\n ... },\n ... return_typemod,\n ... return_type: { name },\n ... }\n ... filter .name = 'std::count';\n {\n Object {\n name: 'std::count',\n annotations: {},\n params: {\n Object {\n kind: 'PositionalParam',\n name: 's',\n num: 0,\n typemod: 'SetOfType',\n type: Object { name: 'anytype' },\n default: {}\n }\n },\n return_typemod: 'SingletonType',\n return_type: Object { name: 'std::int64' }\n }\n }\n\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`Schema > Functions `\n * - :ref:`SDL > Functions `\n * - :ref:`DDL > Functions `\n * - :ref:`Reference > Function calls `\n * - :ref:`Cheatsheets > Functions `\n\n" }, { "path": "docs/reference/datamodel/introspection/index.rst", "content": ".. _ref_datamodel_introspection:\n\nIntrospection\n=============\n\n.. index:: schema module\n.. api-index:: describe, introspect, typeof\n\nAll of the schema information in Gel is stored in the ``schema``\n:ref:`module ` and is accessible via\n*introspection queries*.\n\nAll the introspection types are themselves extending\n:eql:type:`BaseObject`, so they are also subject to introspection :ref:`as\nobject types `. The following\nquery will give a list of all the types used in introspection:\n\n.. code-block:: edgeql\n\n select name := schema::ObjectType.name\n filter name like 'schema::%';\n\nThere's also a couple of ways of getting the introspection type of a\nparticular expression. Any :eql:type:`Object` has a ``__type__`` link\nto the ``schema::ObjectType``. For scalars there's the\n:eql:op:`introspect` and :eql:op:`typeof` operators that can be used\nto get the type of an expression.\n\nFinally, the command :eql:stmt:`describe` can be used to get\ninformation about Gel types in a variety of human-readable formats.\n\n.. toctree::\n :maxdepth: 3\n :hidden:\n\n objects\n scalars\n colltypes\n functions\n triggers\n mutation_rewrites\n indexes\n constraints\n operators\n casts\n" }, { "path": "docs/reference/datamodel/introspection/indexes.rst", "content": ".. _ref_datamodel_introspection_indexes:\n\n=======\nIndexes\n=======\n\nThis section describes introspection of :ref:`indexes\n`.\n\nIntrospection of the ``schema::Index``:\n\n.. code-block:: edgeql-repl\n\n db> with module schema\n ... select ObjectType {\n ... name,\n ... links: {\n ... name,\n ... },\n ... properties: {\n ... name,\n ... }\n ... }\n ... filter .name = 'schema::Index';\n {\n Object {\n name: 'schema::Index',\n links: {Object { name: '__type__' }},\n properties: {\n Object { name: 'expr' },\n Object { name: 'id' },\n Object { name: 'name' }\n }\n }\n }\n\nConsider the following schema:\n\n.. code-block:: sdl\n\n abstract type Addressable {\n property address: str;\n }\n\n type User extending Addressable {\n # define some properties and a link\n required property name: str;\n\n multi link friends: User;\n\n # define an index for User based on name\n index on (.name);\n }\n\nIntrospection of ``User.name`` index:\n\n.. code-block:: edgeql-repl\n\n db> with module schema\n ... select Index {\n ... expr,\n ... }\n ... filter .expr like '%.name';\n {\n Object {\n expr: '.name'\n }\n }\n\nFor introspection of the index within the context of its host type see\n:ref:`object type introspection `.\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`Schema > Indexes `\n * - :ref:`SDL > Indexes `\n * - :ref:`DDL > Indexes `\n" }, { "path": "docs/reference/datamodel/introspection/mutation_rewrites.rst", "content": ".. _ref_datamodel_introspection_mutation_rewrites:\n\n=================\nMutation rewrites\n=================\n\nThis section describes introspection of :ref:`mutation rewrites\n`.\n\nIntrospection of the ``schema::Rewrite``:\n\n.. code-block:: edgeql-repl\n\n db> select schema::ObjectType {\n ... name,\n ... links: {\n ... name\n ... },\n ... properties: {\n ... name\n ... }\n ... } filter .name = 'schema::Rewrite';\n {\n schema::ObjectType {\n name: 'schema::Rewrite',\n links: {\n schema::Link {name: 'subject'},\n schema::Link {name: '__type__'},\n schema::Link {name: 'ancestors'},\n schema::Link {name: 'bases'},\n schema::Link {name: 'annotations'}\n },\n properties: {\n schema::Property {name: 'inherited_fields'},\n schema::Property {name: 'computed_fields'},\n schema::Property {name: 'builtin'},\n schema::Property {name: 'internal'},\n schema::Property {name: 'name'},\n schema::Property {name: 'id'},\n schema::Property {name: 'abstract'},\n schema::Property {name: 'is_abstract'},\n schema::Property {name: 'final'},\n schema::Property {name: 'is_final'},\n schema::Property {name: 'kind'},\n schema::Property {name: 'expr'},\n },\n },\n }\n\nIntrospection of all properties in the ``default`` schema with a mutation\nrewrite:\n\n.. code-block:: edgeql-repl\n\n db> select schema::ObjectType {\n ... name,\n ... properties := (\n ... select .properties {\n ... name,\n ... rewrites: {\n ... kind\n ... }\n ... } filter exists .rewrites\n ... )\n ... } filter .name ilike 'default::%'\n ... and exists .properties.rewrites;\n {\n schema::ObjectType {\n name: 'default::Post',\n properties: {\n schema::Property {\n name: 'created',\n rewrites: {\n schema::Rewrite {\n kind: Insert\n }\n }\n },\n schema::Property {\n name: 'modified',\n rewrites: {\n schema::Rewrite {\n kind: Insert\n },\n schema::Rewrite {\n kind: Update\n }\n }\n },\n },\n },\n }\n\nIntrospection of all rewrites, including the type of query (``kind``),\nrewrite expression, and the object and property they are on:\n\n.. code-block:: edgeql-repl\n\n db> select schema::Rewrite {\n ... subject := (\n ... select .subject {\n ... name,\n ... source: {\n ... name\n ... }\n ... }\n ... ),\n ... kind,\n ... expr\n ... };\n {\n schema::Rewrite {\n subject: schema::Property {\n name: 'created',\n source: schema::ObjectType {\n name: 'default::Post'\n }\n },\n kind: Insert,\n expr: 'std::datetime_of_statement()'\n },\n schema::Rewrite {\n subject: schema::Property {\n name: 'modified',\n source: schema::ObjectType {\n name: 'default::Post'\n }\n },\n kind: Insert,\n expr: 'std::datetime_of_statement()'\n },\n schema::Rewrite {\n subject: schema::Property {\n name: 'modified',\n source: schema::ObjectType {\n name: 'default::Post'\n }\n },\n kind: Update,\n expr: 'std::datetime_of_statement()'\n },\n }\n\nIntrospection of all rewrites on a ``default::Post`` property named\n``modified``:\n\n.. code-block:: edgeql-repl\n\n db> select schema::Rewrite {kind, expr}\n ... filter .subject.source.name = 'default::Post'\n ... and .subject.name = 'modified';\n {\n schema::Rewrite {\n kind: Insert,\n expr: 'std::datetime_of_statement()'\n },\n schema::Rewrite {\n kind: Update,\n expr: 'std::datetime_of_statement()'\n }\n }\n\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`Schema > Mutation rewrites `\n * - :ref:`SDL > Mutation rewrites `\n * - :ref:`DDL > Mutation rewrites `\n\n" }, { "path": "docs/reference/datamodel/introspection/objects.rst", "content": ".. _ref_datamodel_introspection_object_types:\n\n============\nObject types\n============\n\nThis section describes introspection of :ref:`object types\n`.\n\nIntrospection of the ``schema::ObjectType``:\n\n.. code-block:: edgeql-repl\n\n db> with module schema\n ... select ObjectType {\n ... name,\n ... links: {\n ... name,\n ... },\n ... properties: {\n ... name,\n ... }\n ... }\n ... filter .name = 'schema::ObjectType';\n {\n Object {\n name: 'schema::ObjectType',\n links: {\n Object { name: '__type__' },\n Object { name: 'annotations' },\n Object { name: 'bases' },\n Object { name: 'constraints' },\n Object { name: 'indexes' },\n Object { name: 'links' },\n Object { name: 'ancestors' },\n Object { name: 'pointers' },\n Object { name: 'properties' }\n },\n properties: {\n Object { name: 'id' },\n Object { name: 'abstract' },\n Object { name: 'name' }\n }\n }\n }\n\nConsider the following schema:\n\n.. code-block:: sdl\n\n abstract type Addressable {\n address: str;\n }\n\n type User extending Addressable {\n # define some properties and a link\n required name: str;\n\n multi friends: User;\n\n # define an index for User based on name\n index on (.name);\n }\n\nIntrospection of ``User``:\n\n.. code-block:: edgeql-repl\n\n db> with module schema\n ... select ObjectType {\n ... name,\n ... abstract,\n ... bases: { name },\n ... ancestors: { name },\n ... annotations: { name, @value },\n ... links: {\n ... name,\n ... cardinality,\n ... required,\n ... target: { name },\n ... },\n ... properties: {\n ... name,\n ... cardinality,\n ... required,\n ... target: { name },\n ... },\n ... constraints: { name },\n ... indexes: { expr },\n ... }\n ... filter .name = 'default::User';\n {\n Object {\n name: 'default::User',\n abstract: false,\n bases: {Object { name: 'default::Addressable' }},\n ancestors: {\n Object { name: 'std::BaseObject' },\n Object { name: 'std::Object' },\n Object { name: 'default::Addressable' }\n },\n annotations: {},\n links: {\n Object {\n name: '__type__',\n cardinality: 'One',\n required: {},\n target: Object { name: 'schema::Type' }\n },\n Object {\n name: 'friends',\n cardinality: 'Many',\n required: false,\n target: Object { name: 'default::User' }\n }\n },\n properties: {\n Object {\n name: 'address',\n cardinality: 'One',\n required: false,\n target: Object { name: 'std::str' }\n },\n Object {\n name: 'id',\n cardinality: 'One',\n required: true,\n target: Object { name: 'std::uuid' }\n },\n Object {\n name: 'name',\n cardinality: 'One',\n required: true,\n target: Object { name: 'std::str' }\n }\n },\n constraints: {},\n indexes: {\n Object {\n expr: '.name'\n }\n }\n }\n }\n\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`Schema > Object types `\n * - :ref:`SDL > Object types `\n * - :ref:`DDL > Object types `\n * - :ref:`Cheatsheets > Object types `\n\n" }, { "path": "docs/reference/datamodel/introspection/operators.rst", "content": ".. _ref_datamodel_introspection_operators:\n\n=========\nOperators\n=========\n\nThis section describes introspection of Gel operators. Much like\nfunctions, operators have parameters and return types as well as a few\nother features.\n\nIntrospection of the ``schema::Operator``:\n\n.. code-block:: edgeql-repl\n\n db> with module schema\n ... select ObjectType {\n ... name,\n ... links: {\n ... name,\n ... },\n ... properties: {\n ... name,\n ... }\n ... }\n ... filter .name = 'schema::Operator';\n {\n Object {\n name: 'schema::Operator',\n links: {\n Object { name: '__type__' },\n Object { name: 'annotations' },\n Object { name: 'params' },\n Object { name: 'return_type' }\n },\n properties: {\n Object { name: 'id' },\n Object { name: 'name' },\n Object { name: 'operator_kind' },\n Object { name: 'return_typemod' }\n }\n }\n }\n\nSince ``params`` are quite important to operators, here's their structure:\n\n.. code-block:: edgeql-repl\n\n db> with module schema\n ... select ObjectType {\n ... name,\n ... links: {\n ... name,\n ... },\n ... properties: {\n ... name,\n ... }\n ... }\n ... filter .name = 'schema::Parameter';\n {\n Object {\n name: 'schema::Parameter',\n links: {\n Object { name: '__type__' },\n Object { name: 'type' }\n },\n properties: {\n Object { name: 'default' },\n Object { name: 'id' },\n Object { name: 'kind' },\n Object { name: 'name' },\n Object { name: 'num' },\n Object { name: 'typemod' }\n }\n }\n }\n\nIntrospection of the :eql:op:`and` operator:\n\n.. code-block:: edgeql-repl\n\n db> with module schema\n ... select Operator {\n ... name,\n ... operator_kind,\n ... annotations: { name, @value },\n ... params: {\n ... kind,\n ... name,\n ... num,\n ... typemod,\n ... type: { name },\n ... default,\n ... },\n ... return_typemod,\n ... return_type: { name },\n ... }\n ... filter .name = 'std::AND';\n {\n Object {\n name: 'std::AND',\n operator_kind: 'Infix',\n annotations: {},\n params: {\n Object {\n kind: 'PositionalParam',\n name: 'a',\n num: 0,\n typemod: 'SingletonType',\n type: Object { name: 'std::bool' },\n default: {}\n },\n Object {\n kind: 'PositionalParam',\n name: 'b',\n num: 1,\n typemod: 'SingletonType',\n type: Object { name: 'std::bool' },\n default: {}\n }\n },\n return_typemod: 'SingletonType',\n return_type: Object { name: 'std::bool' }\n }\n }\n" }, { "path": "docs/reference/datamodel/introspection/scalars.rst", "content": ".. _ref_datamodel_introspection_scalar_types:\n\n============\nScalar types\n============\n\nThis section describes introspection of :ref:`scalar types\n`.\n\nIntrospection of the ``schema::ScalarType``:\n\n.. code-block:: edgeql-repl\n\n db> with module schema\n ... select ObjectType {\n ... name,\n ... links: {\n ... name,\n ... },\n ... properties: {\n ... name,\n ... }\n ... }\n ... filter .name = 'schema::ScalarType';\n {\n Object {\n name: 'schema::ScalarType',\n links: {\n Object { name: '__type__' },\n Object { name: 'annotations' },\n Object { name: 'bases' },\n Object { name: 'constraints' },\n Object { name: 'ancestors' }\n },\n properties: {\n Object { name: 'default' },\n Object { name: 'enum_values' },\n Object { name: 'id' },\n Object { name: 'abstract' },\n Object { name: 'name' }\n }\n }\n }\n\nIntrospection of the built-in scalar :eql:type:`str`:\n\n.. code-block:: edgeql-repl\n\n db> with module schema\n ... select ScalarType {\n ... name,\n ... default,\n ... enum_values,\n ... abstract,\n ... bases: { name },\n ... ancestors: { name },\n ... annotations: { name, @value },\n ... constraints: { name },\n ... }\n ... filter .name = 'std::str';\n {\n Object {\n name: 'std::str',\n default: {},\n enum_values: {},\n abstract: {},\n bases: {Object { name: 'std::anyscalar' }},\n ancestors: {Object { name: 'std::anyscalar' }},\n annotations: {},\n constraints: {}\n }\n }\n\nFor an :ref:`enumerated scalar type `,\nconsider the following:\n\n.. code-block:: sdl\n\n scalar type Color extending enum;\n\nIntrospection of the enum scalar ``Color``:\n\n.. code-block:: edgeql-repl\n\n db> with module schema\n ... select ScalarType {\n ... name,\n ... default,\n ... enum_values,\n ... abstract,\n ... bases: { name },\n ... ancestors: { name },\n ... annotations: { name, @value },\n ... constraints: { name },\n ... }\n ... filter .name = 'default::Color';\n {\n Object {\n name: 'default::Color',\n default: {},\n enum_values: ['Red', 'Green', 'Blue'],\n abstract: {},\n bases: {Object { name: 'std::anyenum' }},\n ancestors: {\n Object { name: 'std::anyscalar' },\n Object { name: 'std::anyenum' }\n },\n annotations: {},\n constraints: {}\n }\n }\n" }, { "path": "docs/reference/datamodel/introspection/triggers.rst", "content": ".. _ref_datamodel_introspection_triggers:\n\n=========\nTriggers\n=========\n\nThis section describes introspection of :ref:`triggers\n`.\n\nIntrospection of ``schema::Trigger``:\n\n.. code-block:: edgeql-repl\n\n db> with module schema\n ... select ObjectType {\n ... name,\n ... links: {\n ... name,\n ... },\n ... properties: {\n ... name,\n ... }\n ... } filter .name = 'schema::Trigger';\n {\n schema::ObjectType {\n name: 'schema::Trigger',\n links: {\n schema::Link {name: 'subject'},\n schema::Link {name: '__type__'},\n schema::Link {name: 'ancestors'},\n schema::Link {name: 'bases'},\n schema::Link {name: 'annotations'}\n },\n properties: {\n schema::Property {name: 'inherited_fields'},\n schema::Property {name: 'computed_fields'},\n schema::Property {name: 'builtin'},\n schema::Property {name: 'internal'},\n schema::Property {name: 'name'},\n schema::Property {name: 'id'},\n schema::Property {name: 'abstract'},\n schema::Property {name: 'is_abstract'},\n schema::Property {name: 'final'},\n schema::Property {name: 'is_final'},\n schema::Property {name: 'timing'},\n schema::Property {name: 'kinds'},\n schema::Property {name: 'scope'},\n schema::Property {name: 'expr'},\n },\n },\n }\n\nIntrospection of a trigger named ``log_insert`` on the ``User`` type:\n\n.. lint-off\n\n.. code-block:: edgeql-repl\n\n db> with module schema\n ... select Trigger {\n ... name,\n ... kinds,\n ... timing,\n ... scope,\n ... expr,\n ... subject: {\n ... name\n ... }\n ... } filter .name = 'log_insert';\n {\n schema::Trigger {\n name: 'log_insert',\n kinds: {Insert},\n timing: After,\n scope: Each,\n expr: 'insert default::Log { action := \\'insert\\', target_name := __new__.name }',\n subject: schema::ObjectType {name: 'default::User'},\n },\n }\n\n.. lint-on\n\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`Schema > Triggers `\n * - :ref:`SDL > Triggers `\n * - :ref:`DDL > Triggers `\n" }, { "path": "docs/reference/datamodel/linkprops.rst", "content": ".. _ref_datamodel_linkprops:\n\n===============\nLink properties\n===============\n\n.. index:: link property, linkprops, link table, relations\n.. api-index:: @\n\nLinks, like objects, can also contain **properties**. These are used to store metadata about the link. Due to how they're persisted under the hood,\nlink properties have the additional constraint of always being ``single``.\n\nLink properties require non-trivial syntax to use them, so they are considered\nto be an advanced feature. In many cases, regular properties should be used\ninstead. To paraphrase a famous quote: \"Link properties are like a parachute,\nyou don't need them very often, but when you do, they can be clutch.\"\n\n.. note::\n\n In practice, link properties are best used with many-to-many relationships\n (``multi`` links without any exclusive constraints). For one-to-one,\n one-to-many, and many-to-one relationships the same data should be stored in\n object properties instead.\n\n.. versionchanged:: 7.0\n\n Link properties can now be made required.\n\nDeclaration\n===========\n\nLet's a create a ``Person.friends`` link with a ``strength`` property\ncorresponding to the strength of the friendship.\n\n.. code-block:: sdl\n\n type Person {\n required name: str { constraint exclusive };\n\n multi friends: Person {\n strength: float64;\n }\n }\n\nConstraints\n===========\n\nNow let's ensure that the ``@strength`` property is always non-negative:\n\n.. code-block:: sdl\n\n type Person {\n required name: str { constraint exclusive };\n\n multi friends: Person {\n strength: float64;\n\n constraint expression on (\n __subject__@strength >= 0\n );\n }\n }\n\nIndexes\n=======\n\nTo add an index on a link property, we have to refactor our code and define\nan abstract link ``friendship`` that will contain the ``strength`` property\nwith an index on it:\n\n\n.. code-block:: sdl\n\n abstract link friendship {\n required strength: float64;\n index on (__subject__@strength);\n }\n\n type Person {\n required name: str { constraint exclusive };\n\n multi friends: Person {\n extending friendship;\n };\n }\n\nConceptualizing link properties\n===============================\n\nA way to conceptualize the difference between a regular property and\na link property is that regular properties are used to construct an object,\nwhile link properties are used to construct the link between objects.\n\nFor example, here the ``name`` and ``email`` properties are used to construct a\n``Person`` object:\n\n.. code-block:: edgeql\n\n insert Person {\n name := \"Jane\",\n email := \"jane@jane.com\"\n }\n\nNow let's insert a ``Person`` object linking it to another ``Person`` object\nsetting the ``@strength`` property to the link between them:\n\n.. code-block:: edgeql\n\n insert Person {\n name := \"Bob\",\n email := \"bob@bob.com\",\n friends := (\n insert Person {\n name := \"Jane\",\n email := \"jane@jane.com\",\n @strength := 3.14\n }\n )\n }\n\nSo we're not using ``@strength`` to construct a particular ``Person`` object,\nbut to quantify a link between two ``Person`` objects.\n\nInserting\n=========\n\nWhat if we want to insert a ``Person`` object while linking it to another\n``Person`` that's already in the database?\n\nThe ``@strength`` property then will be specified in the *shape* of a\n``select`` subquery:\n\n.. code-block:: edgeql\n\n insert Person {\n name := \"Bob\",\n friends := (\n select detached Person {\n @strength := 3.14\n }\n filter .name = \"Alice\"\n )\n }\n\n.. note::\n\n We are using the :eql:op:`detached` operator to unbind the\n ``Person`` reference from the scope of the ``insert`` query.\n\nWhen doing a nested insert, link properties can be directly included in the\ninner ``insert`` subquery:\n\n.. code-block:: edgeql\n\n insert Person {\n name := \"Bob\",\n friends := (\n insert Person {\n name := \"Jane\",\n @strength := 3.14\n }\n )\n }\n\nSimilarly, ``with`` can be used to capture an expression returning an\nobject type, after which a link property can be added when linking it to\nanother object type:\n\n.. code-block:: edgeql\n\n with\n alice := (\n\n insert Person {\n name := \"Alice\"\n }\n unless conflict on .name\n else (\n select Person\n filter .name = \"Alice\" limit 1\n )\n )\n\n insert Person {\n name := \"Bob\",\n friends := alice {\n @strength := 3.14\n }\n };\n\nUpdating\n========\n\n.. code-block:: edgeql\n\n update Person\n filter .name = \"Bob\"\n set {\n friends += (\n select .friends {\n @strength := 3.7\n }\n filter .name = \"Alice\"\n )\n };\n\nThe example updates the ``@strength`` property of Bob's friends link to\nAlice to 3.7.\n\nIn the context of multi links the ``+=`` operator works like an\nan insert/update operator.\n\nTo update one or more links in a multi link, you can select from the current\nlinked objects, as the example does. Use a ``detached`` selection if you\nwant to insert/update a wider selection of linked objects instead.\n\n\nSelecting\n=========\n\nTo select a link property, you can use the ``@<>name`` syntax inside the\nselect *shape*. Keep in mind, that you're not selecting a property on\nan object with this syntax, but rather on the link, in this case ``friends``:\n\n.. code-block:: edgeql-repl\n\n gel> select Person {\n .... name,\n .... friends: {\n .... name,\n .... @strength\n .... }\n .... };\n {\n default::Person {name: 'Alice', friends: {}},\n default::Person {\n name: 'Bob',\n friends: {\n default::Person {name: 'Alice', @strength: 3.7}\n }\n },\n }\n\nUnions\n======\n\nA link property cannot be referenced in a set union *except* in the case of\na :ref:`for loop `. That means this will *not* work:\n\n.. code-block:: edgeql\n\n # šŸš« Does not work\n insert Movie {\n title := 'The Incredible Hulk',\n actors := {(\n select Person {\n @character_name := 'The Hulk'\n } filter .name = 'Mark Ruffalo'\n ),\n (\n select Person {\n @character_name := 'Iron Man'\n } filter .name = 'Robert Downey Jr.'\n )}\n };\n\nThat query will produce an error: ``QueryError: invalid reference to link\nproperty in top level shape``\n\nYou can use this workaround instead:\n\n.. code-block:: edgeql\n\n # āœ… Works!\n insert Movie {\n title := 'The Incredible Hulk',\n\n actors := assert_distinct((\n with characters := {\n ('The Hulk', 'Mark Ruffalo'),\n ('Iron Man', 'Robert Downey Jr.')\n }\n for character in characters union (\n select Person {\n @character_name := character.0\n } filter .name = character.1\n )\n ))\n };\n\nNote that we are also required to wrap the ``actors`` query with\n:eql:func:`assert_distinct` here to assure the compiler that the result set\nis distinct.\n\nWith computed backlinks\n=======================\n\nSpecifying link properties of a computed backlink in your shape is also\nsupported. If you have this schema:\n\n.. code-block:: sdl\n\n type Person {\n required name: str;\n\n multi follows: Person {\n followed: datetime {\n default := datetime_of_statement();\n };\n };\n\n multi link followers := .`\n * - :ref:`Properties in schema `\n * - :ref:`Properties with DDL `\n" }, { "path": "docs/reference/datamodel/links.rst", "content": ".. _ref_datamodel_links:\n\n=====\nLinks\n=====\n\nLinks define a relationship between two\n:ref:`object types ` in Gel.\n\nLinks in |Gel| are incredibly powerful and flexible. They can be used to model\nrelationships of any cardinality, can be traversed in both directions,\ncan be polymorphic, can have constraints, and many other things.\n\n\nLinks are directional\n=====================\n\nLinks are *directional*: they have a **source** (the type on which they are\ndeclared) and a **target** (the type they point to).\n\nE.g. the following schema defines a link from ``Person`` to ``Person`` and\na link from ``Company`` to ``Person``:\n\n.. code-block:: sdl\n\n type Person {\n link best_friend: Person;\n }\n\n type Company {\n multi link employees: Person;\n }\n\nThe ``employees`` link's source is ``Company`` and its target is ``Person``.\n\nThe ``link`` keyword is optional, and can be omitted.\n\n\nLink cardinality\n================\n\n.. api-index:: single, multi\n\nAll links have a cardinality: either ``single`` or ``multi``. The default is\n``single`` (a \"to-one\" link). Use the ``multi`` keyword to declare a \"to-many\"\nlink:\n\n.. code-block:: sdl\n\n type Person {\n multi friends: Person;\n }\n\n\nRequired links\n==============\n\n.. index:: not null\n.. api-index:: required, optional\n\nAll links are either ``optional`` or ``required``; the default is ``optional``.\nUse the ``required`` keyword to declare a required link. A required link must\npoint to *at least one* target instance, and if the cardinality of the required\nlink is ``single``, it must point to *exactly one* target instance. In this\nscenario, every ``Person`` must have *exactly one* ``best_friend``:\n\n.. code-block:: sdl\n\n type Person {\n required best_friend: Person;\n }\n\nLinks with cardinality ``multi`` can also be ``required``;\n``required multi`` links must point to *at least one* target object:\n\n.. code-block:: sdl\n\n type Person {\n name: str;\n }\n\n type GroupChat {\n required multi members: Person;\n }\n\nAttempting to create a ``GroupChat`` with no members would fail.\n\nExclusive constraints\n=====================\n\n.. api-index:: constraint exclusive\n\nYou can add an ``exclusive`` constraint to a link to guarantee that no other\ninstances can link to the same target(s):\n\n.. code-block:: sdl\n\n type Person {\n name: str;\n }\n\n type GroupChat {\n required multi members: Person {\n constraint exclusive;\n }\n }\n\nWith ``exclusive`` on ``GroupChat.members``, two ``GroupChat`` objects cannot\nlink to the same ``Person``; put differently, no ``Person`` can be a\n``member`` of multiple ``GroupChat`` objects.\n\nBacklinks\n=========\n\n.. api-index:: .<\n\nIn Gel you can traverse links in reverse to find objects that link to\nthe object. You can do that directly in your query. E.g. for this example\nschema:\n\n.. code-block:: sdl\n\n type Author {\n name: str;\n }\n\n type Article {\n title: str;\n multi authors: Author;\n }\n\nYou can find all articles by \"John Doe\" by traversing the ``authors``\nlink in reverse:\n\n.. code-block:: edgeql\n\n select Author {\n articles := .` to\nlearn more.\n\n\n.. _ref_guide_one_to_many:\n\nOne-to-many\n-----------\n\nConceptually, one-to-many and many-to-one relationships are identical; the\n\"directionality\" is a matter of perspective. Here, the same \"shirt owner\"\nrelationship is represented with a ``multi`` link:\n\n.. code-block:: sdl\n\n type Person {\n required name: str;\n multi shirts: Shirt {\n # ensures a one-to-many relationship\n constraint exclusive;\n }\n }\n\n type Shirt {\n required color: str;\n }\n\n.. note::\n\n Don't forget the ``exclusive`` constraint! Without it, the relationship\n becomes many-to-many.\n\nUnder the hood, a ``multi`` link is stored in an intermediate `association\ntable `_, whereas a\n``single`` link is stored as a column in the object type where it is declared.\n\n.. note::\n\n Choosing a link direction can be tricky. Should you model this\n relationship as one-to-many (with a ``multi`` link) or as many-to-one\n (with a ``single`` link and a backlink)? A general rule of thumb:\n\n - Use a ``multi`` link if the relationship is relatively stable and\n not updated frequently, and the set of related objects is typically\n small. For example, a list of postal addresses in a user profile.\n - Otherwise, prefer a single link from one object type and a computed\n backlink on the other. This can be more efficient and is generally\n recommended for 1:N relations:\n\n .. code-block:: sdl\n\n type Post {\n required author: User;\n }\n\n type User {\n multi posts := (.;\n };\n }\n type Movie {\n required title: str;\n }\n\n Alternatively, you might introduce a dedicated type:\n\n .. code-block:: sdl\n\n type User {\n required name: str;\n multi watch_history := .`.\n\nInserting and updating link properties\n--------------------------------------\n\nTo add a link with a link property, include the property name (prefixed by\n``@``) in the shape:\n\n.. code-block:: edgeql\n\n insert Person {\n name := \"Bob\",\n family_members := (\n select detached Person {\n @relationship := \"sister\"\n }\n filter .name = \"Alice\"\n )\n };\n\nUpdating a link's property on an **existing** link is similar. You can select\nthe link from within the object being updated:\n\n.. code-block:: edgeql\n\n update Person\n filter .name = \"Bob\"\n set {\n family_members := (\n select .family_members {\n @relationship := \"step-sister\"\n }\n filter .name = \"Alice\"\n )\n };\n\n.. warning::\n\n A link property cannot be referenced in a set union *except* in the case of\n a :ref:`for loop `. For instance:\n\n .. code-block:: edgeql\n\n # šŸš« Does not work\n insert Movie {\n title := 'The Incredible Hulk',\n characters := {\n (\n select Person {\n @character_name := 'The Hulk'\n }\n filter .name = 'Mark Ruffalo'\n ),\n (\n select Person {\n @character_name := 'Abomination'\n }\n filter .name = 'Tim Roth'\n )\n }\n };\n\n will produce an error ``QueryError: invalid reference to link property in\n top level shape``.\n\n One workaround is to insert them via a ``for`` loop, combined with\n :eql:func:`assert_distinct`:\n\n .. code-block:: edgeql\n\n # āœ… Works!\n insert Movie {\n title := 'The Incredible Hulk',\n characters := assert_distinct((\n with actors := {\n ('The Hulk', 'Mark Ruffalo'),\n ('Abomination', 'Tim Roth')\n },\n for actor in actors union (\n select Person {\n @character_name := actor.0\n }\n filter .name = actor.1\n )\n ))\n };\n\nQuerying link properties\n------------------------\n\nTo query a link property, add the link property's name (prefixed with ``@``)\nin the shape:\n\n.. code-block:: edgeql-repl\n\n db> select Person {\n ... name,\n ... family_members: {\n ... name,\n ... @relationship\n ... }\n ... };\n\n.. note::\n\n In the results above, Bob has a *step-sister* property on the link to\n Alice, but Alice does not automatically have a property describing Bob.\n Changes to link properties are not mirrored on the \"backlink\" side unless\n explicitly updated, because link properties cannot be required.\n\n.. note::\n\n For a full guide on modeling, inserting, updating, and querying link\n properties, see the :ref:`Using Link Properties `\n guide.\n\n\n.. _ref_datamodel_link_deletion:\n\nDeletion policies\n=================\n\n.. api-index:: on target delete, on source delete, restrict, delete source,\n allow, deferred restrict, delete target, if orphan\n\nLinks can declare their own **deletion policy** for when the **target** or\n**source** is deleted.\n\nTarget deletion\n---------------\n\nThe clause ``on target delete`` determines the action when the target object is\ndeleted:\n\n- ``restrict`` (default) ā€” raises an exception if the target is deleted.\n- ``delete source`` ā€” deletes the source when the target is deleted (a cascade).\n- ``allow`` ā€” removes the target from the link if the target is deleted.\n- ``deferred restrict`` ā€” like ``restrict`` but defers the error until the\n end of the transaction if the object remains linked.\n\n.. code-block:: sdl\n\n type MessageThread {\n title: str;\n }\n\n type Message {\n content: str;\n chat: MessageThread {\n on target delete delete source;\n }\n }\n\n\n.. _ref_datamodel_links_source_deletion:\n\nSource deletion\n---------------\n\nThe clause ``on source delete`` determines the action when the **source** is\ndeleted:\n\n- ``allow`` ā€” deletes the source, removing the link to the target.\n- ``delete target`` ā€” unconditionally deletes the target as well.\n- ``delete target if orphan`` ā€” deletes the target if and only if it's no\n longer linked by any other object *via the same link*.\n\n.. code-block:: sdl\n\n type MessageThread {\n title: str;\n multi messages: Message {\n on source delete delete target;\n }\n }\n\n type Message {\n content: str;\n }\n\nYou can add ``if orphan`` if you'd like to avoid deleting a target that remains\nlinked elsewhere via the **same** link name.\n\n.. code-block:: sdl-diff\n\n type MessageThread {\n title: str;\n multi messages: Message {\n - on source delete delete target;\n + on source delete delete target if orphan;\n }\n }\n\n.. note::\n\n The ``if orphan`` qualifier **does not** apply globally across\n all links in the database or even all links from the same type. If another\n link *by a different name* or *with a different on-target-delete* policy\n points at the same object, it *doesn't* prevent the object from being\n considered \"orphaned\" for the link that includes ``if orphan``.\n\n\n.. _ref_datamodel_link_polymorphic:\n\nPolymorphic links\n=================\n\nLinks can be **polymorphic**, i.e., have an ``abstract`` target. In the\nexample below, we have an abstract type ``Person`` with concrete subtypes\n``Hero`` and ``Villain``:\n\n.. code-block:: sdl\n\n abstract type Person {\n name: str;\n }\n\n type Hero extending Person {\n # additional fields\n }\n\n type Villain extending Person {\n # additional fields\n }\n\nA polymorphic link can target any non-abstract subtype:\n\n.. code-block:: sdl\n\n type Movie {\n title: str;\n multi characters: Person;\n }\n\nWhen querying a polymorphic link, you can filter by a specific subtype, cast\nthe link to a subtype, etc. See :ref:`Polymorphic Queries `\nfor details.\n\nAbstract links\n==============\n\n.. api-index:: abstract link\n\nIt's possible to define ``abstract`` links that aren't tied to a particular\nsource or target, and then extend them in concrete object types. This can help\neliminate repetitive declarations:\n\n.. code-block:: sdl\n\n abstract link link_with_strength {\n strength: float64;\n index on (__subject__@strength);\n }\n\n type Person {\n multi friends: Person {\n extending link_with_strength;\n };\n }\n\n\n.. _ref_eql_sdl_links_overloading:\n\nOverloading\n===========\n\n.. api-index:: overloaded\n\nWhen an inherited link is modified (by adding more constraints or changing its\ntarget type, etc.), the ``overloaded`` keyword is required. This prevents\nunintentional overloading due to name clashes:\n\n.. code-block:: sdl\n\n abstract type Friendly {\n # this type can have \"friends\"\n multi friends: Friendly;\n }\n\n type User extending Friendly {\n # overload the link target to to be specifically User\n overloaded multi friends: User;\n\n # ... other links and properties\n }\n\n\n.. _ref_eql_sdl_links:\n.. _ref_eql_sdl_links_syntax:\n\nDeclaring links\n===============\n\nThis section describes the syntax to use links in your schema.\n\nSyntax\n------\n\n.. sdl:synopsis::\n\n # Concrete link form used inside type declaration:\n [ overloaded ] [{required | optional}] [{single | multi}]\n [ link ] : \n [ \"{\"\n [ extending [, ...] ; ]\n [ default := ; ]\n [ readonly := {true | false} ; ]\n [ on target delete ; ]\n [ on source delete ; ]\n [ ]\n [ ]\n [ ]\n ...\n \"}\" ]\n\n # Computed link form used inside type declaration:\n [{required | optional}] [{single | multi}]\n [ link ] := ;\n\n # Computed link form used inside type declaration (extended):\n [ overloaded ] [{required | optional}] [{single | multi}]\n link [: ]\n [ \"{\"\n using () ;\n [ extending [, ...] ; ]\n [ ]\n [ ]\n ...\n \"}\" ]\n\n # Abstract link form:\n abstract link \n [ \"{\"\n [ extending [, ...] ; ]\n [ readonly := {true | false} ; ]\n [ ]\n [ ]\n [ ]\n [ ]\n ...\n \"}\" ]\n\nThere are several forms of link declaration, as shown in the syntax synopsis\nabove:\n\n- the first form is the canonical definition form;\n- the second form is used for defining a\n :ref:`computed link `;\n- and the last form is used to define an abstract link.\n\nThe following options are available:\n\n:eql:synopsis:`overloaded`\n If specified, indicates that the link is inherited and that some\n feature of it may be altered in the current object type. It is an\n error to declare a link as *overloaded* if it is not inherited.\n\n:eql:synopsis:`required`\n If specified, the link is considered *required* for the parent\n object type. It is an error for an object to have a required\n link resolve to an empty value. Child links **always** inherit\n the *required* attribute, i.e it is not possible to make a\n required link non-required by extending it.\n\n:eql:synopsis:`optional`\n This is the default qualifier assumed when no qualifier is\n specified, but it can also be specified explicitly. The link is\n considered *optional* for the parent object type, i.e. it is\n possible for the link to resolve to an empty value.\n\n:eql:synopsis:`multi`\n Specifies that there may be more than one instance of this link\n in an object, in other words, ``Object.link`` may resolve to a set\n of a size greater than one.\n\n:eql:synopsis:`single`\n Specifies that there may be at most *one* instance of this link\n in an object, in other words, ``Object.link`` may resolve to a set\n of a size not greater than one. ``single`` is assumed if nether\n ``multi`` nor ``single`` qualifier is specified.\n\n:eql:synopsis:`extending [, ...]`\n Optional clause specifying the *parents* of the new link item.\n\n Use of ``extending`` creates a persistent schema relationship\n between the new link and its parents. Schema modifications\n to the parent(s) propagate to the child.\n\n If the same *property* name exists in more than one parent, or\n is explicitly defined in the new link and at least one parent,\n then the data types of the property targets must be *compatible*.\n If there is no conflict, the link properties are merged to form a\n single property in the new link item.\n\n:eql:synopsis:``\n The type must be a valid :ref:`type expression `\n denoting an object type.\n\nThe valid SDL sub-declarations are listed below:\n\n:eql:synopsis:`default := `\n Specifies the default value for the link as an EdgeQL expression.\n The default value is used in an ``insert`` statement if an explicit\n value for this link is not specified.\n\n The expression must be :ref:`Stable `.\n\n:eql:synopsis:`readonly := {true | false}`\n If ``true``, the link is considered *read-only*. Modifications of this link using ``update`` are prohibited once an object is created. Any :ref:`overloaded links ` **must** preserve the original *read-only* value. Changes to this link **will** occur if a link is deleted and the appropriate :ref:`deletion policy ` allows it.\n\n:sdl:synopsis:``\n Set link :ref:`annotation `\n to a given *value*.\n\n:sdl:synopsis:``\n Define a concrete :ref:`property ` on the link.\n\n:sdl:synopsis:``\n Define a concrete :ref:`constraint ` on the link.\n\n:sdl:synopsis:``\n Define an :ref:`index ` for this abstract\n link. Note that this index can only refer to link properties.\n\n\n.. _ref_eql_ddl_links:\n\nDDL commands\n============\n\nThis section describes the low-level DDL commands for creating, altering, and\ndropping links. You typically don't need to use these commands directly, but\nknowing about them is useful for reviewing migrations.\n\nCreate link\n-----------\n\n:eql-statement:\n:eql-haswith:\n\nDefine a new link.\n\n.. eql:synopsis::\n\n [ with [, ...] ]\n {create|alter} type \"{\"\n [ ... ]\n create [{required | optional}] [{single | multi}]\n link \n [ extending [, ...] ]: \n [ \"{\" ; [...] \"}\" ] ;\n [ ... ]\n \"}\"\n\n # Computed link form:\n\n [ with [, ...] ]\n {create|alter} type \"{\"\n [ ... ]\n create [{required | optional}] [{single | multi}]\n link := ;\n [ ... ]\n \"}\"\n\n # Abstract link form:\n\n [ with [, ...] ]\n create abstract link [::] [extending [, ...]]\n [ \"{\" ; [...] \"}\" ]\n\n # where is one of\n\n set default := \n set readonly := {true | false}\n create annotation := \n create property ...\n create constraint ...\n on target delete \n on source delete \n reset on target delete\n create index on \n\nDescription\n^^^^^^^^^^^\n\nThe combinations of ``create type ... create link`` and ``alter type ...\ncreate link`` define a new concrete link for a given object type, in DDL form.\n\nThere are three forms of ``create link``:\n\n1. The canonical definition form (specifying a target type).\n2. The computed link form (declaring a link via an expression).\n3. The abstract link form (declaring a module-level link).\n\nParameters\n^^^^^^^^^^^\n\nMost sub-commands and options mirror those found in the\n:ref:`SDL link declaration `. In DDL form:\n\n- ``set default := `` specifies a default value.\n- ``set readonly := {true | false}`` makes the link read-only or not.\n- ``create annotation := `` adds an annotation.\n- ``create property ...`` defines a property on the link.\n- ``create constraint ...`` defines a constraint on the link.\n- ``on target delete `` and ``on source delete `` specify\n deletion policies.\n- ``reset on target delete`` resets the target deletion policy to default\n or inherited.\n- ``create index on `` creates an index on the link.\n\nExamples\n^^^^^^^^\n\n.. code-block:: edgeql\n\n alter type User {\n create multi link friends: User\n };\n\n.. code-block:: edgeql\n\n alter type User {\n create link special_group := (\n select __source__.friends\n filter .town = __source__.town\n )\n };\n\n.. code-block:: edgeql\n\n create abstract link orderable {\n create property weight: std::int64\n };\n\n alter type User {\n create multi link interests extending orderable: Interest\n };\n\n\nAlter link\n----------\n\n:eql-statement:\n:eql-haswith:\n\nChanges the definition of a link.\n\n.. eql:synopsis::\n\n [ with [, ...] ]\n {create|alter} type \"{\"\n [ ... ]\n alter link \n [ \"{\" ] ; [...] [ \"}\" ];\n [ ... ]\n \"}\"\n\n [ with [, ...] ]\n alter abstract link [::]\n [ \"{\" ] ; [...] [ \"}\" ];\n\n # where is one of\n\n set default := \n reset default\n set readonly := {true | false}\n reset readonly\n rename to \n extending ...\n set required\n set optional\n reset optionality\n set single\n set multi\n reset cardinality\n set type [using ()]\n reset type\n using ()\n create annotation := \n alter annotation := \n drop annotation \n create property ...\n alter property ...\n drop property ...\n create constraint ...\n alter constraint ...\n drop constraint ...\n on target delete \n on source delete \n create index on \n drop index on \n\nDescription\n^^^^^^^^^^^\n\nThis command modifies an existing link on a type. It can also be used on\nan abstract link at the module level.\n\nParameters\n^^^^^^^^^^\n\n- ``rename to `` changes the link's name.\n- ``extending ...`` changes or adds link parents.\n- ``set required`` / ``set optional`` changes the link optionality.\n- ``reset optionality`` reverts optionality to default or inherited value.\n- ``set single`` / ``set multi`` changes cardinality.\n- ``reset cardinality`` reverts cardinality to default or inherited value.\n- ``set type [using ()]`` changes the link's target type.\n- ``reset type`` reverts the link's type to inherited.\n- ``using ()`` changes the expression of a computed link.\n- ``create annotation``, ``alter annotation``, ``drop annotation`` manage\n annotations.\n- ``create property``, ``alter property``, ``drop property`` manage link\n properties.\n- ``create constraint``, ``alter constraint``, ``drop constraint`` manage\n link constraints.\n- ``on target delete `` and ``on source delete `` manage\n deletion policies.\n- ``reset on target delete`` reverts the target deletion policy.\n- ``create index on `` / ``drop index on `` manage\n indexes on link properties.\n\nExamples\n^^^^^^^^\n\n.. code-block:: edgeql\n\n alter type User {\n alter link friends create annotation title := \"Friends\";\n };\n\n.. code-block:: edgeql\n\n alter abstract link orderable rename to sorted;\n\n.. code-block:: edgeql\n\n alter type User {\n alter link special_group using (\n # at least one of the friend's interests\n # must match the user's\n select __source__.friends\n filter .interests IN __source__.interests\n );\n };\n\nDrop link\n---------\n\n:eql-statement:\n:eql-haswith:\n\nRemoves the specified link from the schema.\n\n.. eql:synopsis::\n\n [ with [, ...] ]\n alter type \"{\"\n [ ... ]\n drop link \n [ ... ]\n \"}\"\n\n [ with [, ...] ]\n drop abstract link []::\n\nDescription\n^^^^^^^^^^^\n\n- ``alter type ... drop link `` removes the link from an object type.\n- ``drop abstract link `` removes an abstract link from the schema.\n\nExamples\n^^^^^^^^\n\n.. code-block:: edgeql\n\n alter type User drop link friends;\n\n.. code-block:: edgeql\n\n drop abstract link orderable;\n\n\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n - :ref:`Introspection > Object types `\n" }, { "path": "docs/reference/datamodel/migrations.rst", "content": ".. _ref_datamodel_migrations:\n\n==========\nMigrations\n==========\n\n|Gel's| baked-in migration system lets you painlessly evolve your schema over\ntime. Just update the contents of your |.gel| file(s) and use the |Gel| CLI\nto *create* and *apply* migrations.\n\n.. code-block:: bash\n\n $ gel migration create\n Created dbschema/migrations/00001.edgeql\n\n $ gel migrate\n Applied dbschema/migrations/00001.edgeql\n\nRefer to the :ref:`creating and applying migrations `\nguide for more information on how to use the migration system.\n\nThis document describes how migrations are implemented.\n\n\nThe migrations flow\n===================\n\nThe migration flow is as follows:\n\n1. The user edits the |.gel| files in the ``dbschema`` directory.\n\n This makes the schema described in the |.gel| files **different** from the\n actual schema in the database.\n\n2. The user runs the :gelcmd:`migration create` command to create a new\n migration (a sequence of low-level DDL commands).\n\n * The CLI reads the |.gel| files and sends them to the |Gel| server, to\n analyze the changes.\n\n * The |Gel| server generates a migration plan and sends it back to the CLI.\n\n * The migration plan might require clarification from the user.\n\n If so, the CLI and the |Gel| server will go back and forth presenting\n the user with a sequence of questions, until the migration plan is\n clear and approved by the user.\n\n3. The CLI writes the migration plan to a new file in the ``dbschema/migrations``\n directory.\n\n4. The user runs the :gelcmd:`migrate` command to apply the migration to the\n database.\n\n5. The user checks in the updated |.gel| files and the new\n ``dbschema/migrations`` migration file (created by :gelcmd:`migration create`)\n into version control.\n\n\nCommand line tools\n==================\n\nThe two most important commands are:\n\n* :gelcmd:`migration create`\n* :gelcmd:`migrate`\n\n\nAutomatic migrations\n====================\n\nSometimes when you're prototyping something new you don't want to spend\ntime worrying about migrations. There's no data to lose and not much code\nthat depends on the schema just yet.\n\nFor this use case you can use the :gelcmd:`watch --migrate` command, which\nwill monitor your |.gel| files and automatically create and apply migrations\nfor you in the background.\n\n.. _ref_eql_ddl:\n\nData definition language (DDL)\n==============================\n\nThe migration plan is a sequence of DDL commands. DDL commands are low-level\ninstructions that describe the changes to the schema.\n\nSDL and your |.gel| files are like a 3D printer: you design the final shape,\nand the system puts a database together for you. Using DDL is like building a\nhouse the traditional way: to add a window, you first need a frame; to have a\nframe, you need a wall; and so on.\n\nIf your schema looks like this:\n\n.. code-block:: sdl\n\n type User {\n required name: str;\n }\n\nthen the corresponding DDL might look like this:\n\n.. code-block:: edgeql\n\n create type User {\n create required property name: str;\n }\n\nThere are some circumstances where users might want to use DDL directly.\nBut in most cases you just need to learn how to read them to understand\nthe migration plan. Luckily, the DDL and SDL syntaxes were designed in tandem\nand are very similar.\n\nMost documentation pages on Gel's schema have a section about DDL commands,\ne.g. :ref:`object types DDL `.\n\n\n.. _ref_eql_ddl_migrations:\n\nMigration DDL commands\n======================\n\nMigrations themselves are a sequence of special DDL commands.\n\nLike all DDL commands, ``start migration`` and other migration commands are\nconsidered low-level. Users are encouraged to use the built-in\n:ref:`migration tools ` instead.\n\nHowever, if you want to implement your own migration tools, this section\nwill give you a good understanding of how Gel migrations work under the hood.\n\n\nStart migration\n---------------\n\n:eql-statement:\n\nStart a migration block.\n\n.. eql:synopsis::\n\n start migration to \"{\"\n ;\n [ ... ]\n \"}\" ;\n\nParameters\n^^^^^^^^^^\n\n:eql:synopsis:``\n Complete schema text (content of all |.gel| files) defined with\n the declarative :ref:`Gel schema definition language `.\n\nDescription\n^^^^^^^^^^^\n\nThe command ``start migration`` defines a migration of the schema to a\nnew state. The target schema state is described using :ref:`SDL\n` and describes the entire schema. This is important to\nremember when creating a migration to add a few more things to an\nexisting schema as all the existing schema objects and the new ones\nmust be included in the ``start migration`` command. Objects that\naren't included in the command will be removed from the new schema\n(which may result in data loss).\n\nThis command also starts a transaction block if not inside a\ntransaction already.\n\nWhile inside a migration block, all issued EdgeQL statements are not executed\nimmediately and are instead recorded to be part of the migration script. Aside\nfrom normal EdgeQL commands the following special migration commands are\navailable:\n\n* :eql:stmt:`describe current migration` -- return a list of statements\n currently recorded as part of the migration;\n\n* :eql:stmt:`populate migration` -- auto-populate the migration with\n system-generated DDL statements to achieve the target schema state;\n\n* :eql:stmt:`abort migration` -- abort the migration block and discard the\n migration;\n\n* :eql:stmt:`commit migration` -- commit the migration by executing the\n migration script statements and recording the migration into the system\n migration log.\n\nExample\n^^^^^^^\n\nCreate a new migration to a target schema specified by the Gel Schema\nsyntax:\n\n.. code-block:: edgeql\n\n start migration to {\n module default {\n type User {\n property username: str;\n };\n };\n };\n\n\n.. _ref_eql_ddl_migrations_create:\n\ncreate migration\n----------------\n\n:eql-statement:\n\nCreate a new migration using an explicit EdgeQL script.\n\n.. eql:synopsis::\n\n create migration \"{\"\n ;\n [ ... ]\n \"}\" ;\n\nParameters\n^^^^^^^^^^\n\n:eql:synopsis:``\n Any valid EdgeQL statement, except ``database``, ``branch``, ``role``,\n ``configure``, ``migration``, or ``transaction`` statements.\n\nDescription\n^^^^^^^^^^^\n\nThe command ``create migration`` executes all the nested EdgeQL commands\nand records the migration into the system migration log.\n\nExample\n^^^^^^^\n\nCreate a new migration to a target schema specified by the Gel Schema\nsyntax:\n\n.. code-block:: edgeql\n\n create migration {\n create type default::User {\n create property username: str;\n }\n };\n\n\nAbort migration\n---------------\n\n:eql-statement:\n\nAbort the current migration block and discard the migration.\n\n.. eql:synopsis::\n\n abort migration ;\n\nDescription\n^^^^^^^^^^^\n\nThe command ``abort migration`` is used to abort a migration block started by\n:eql:stmt:`start migration`. Issuing ``abort migration`` outside of a\nmigration block is an error.\n\nExample\n^^^^^^^\n\nStart a migration block and then abort it:\n\n.. code-block:: edgeql\n\n start migration to {\n module default {\n type User;\n };\n };\n\n abort migration;\n\n\nPopulate migration\n------------------\n\n:eql-statement:\n\nPopulate the current migration with system-generated statements.\n\n.. eql:synopsis::\n\n populate migration ;\n\nDescription\n^^^^^^^^^^^\n\nThe command ``populate migration`` is used within a migration block started by\n:eql:stmt:`start migration` to automatically fill the migration with\nsystem-generated statements to achieve the desired target schema state. If\nthe system is unable to automatically find a satisfactory sequence of\nstatements to perform the migration, an error is returned. Issuing ``populate\nmigration`` outside of a migration block is also an error.\n\n.. warning::\n\n The statements generated by ``populate migration`` may drop schema objects,\n which may result in data loss. Make sure to inspect the generated\n migration using :eql:stmt:`describe current migration` before running\n :eql:stmt:`commit migration`!\n\nExample\n^^^^^^^\n\nStart a migration block and populate it with auto-generated statements.\n\n.. code-block:: edgeql\n\n start migration to {\n module default {\n type User;\n };\n };\n\n populate migration;\n\n\nDescribe current migration\n--------------------------\n\n:eql-statement:\n\nDescribe the migration in the current migration block.\n\n.. eql:synopsis::\n\n describe current migration [ as {ddl | json} ];\n\n\nDescription\n^^^^^^^^^^^\n\nThe command ``describe current migration`` generates a description of\nthe migration in the current migration block in the specified output\nformat:\n\n:eql:synopsis:`as ddl`\n Show a sequence of statements currently recorded as part of the migration\n using valid :ref:`DDL ` syntax. The output will indicate\n if the current migration is fully defined, i.e. the recorded statements\n bring the schema to the state specified by :eql:stmt:`start migration`.\n\n:eql:synopsis:`as json`\n Provide a machine-readable description of the migration using the following\n JSON format:\n\n .. code-block::\n\n {\n // Name of the parent migration\n \"parent\": \"\",\n\n // Whether the confirmed DDL makes the migration complete,\n // i.e. there are no more statements to issue.\n \"complete\": {true|false},\n\n // List of confirmed migration statements\n \"confirmed\": [\n \"\",\n ...\n ],\n\n // The variants of the next statement\n // suggested by the system to advance\n // the migration script.\n \"proposed\": {\n \"statements\": [{\n \"text\": \"\"\n }],\n \"required-user-input\": [\n {\n \"placeholder\": \"\",\n \"prompt\": \"\"\n },\n ...\n ],\n \"confidence\": (0..1), // confidence coefficient\n \"prompt\": \"\",\n \"prompt_id\": \"\",\n // Whether the operation is considered to be non-destructive.\n \"data_safe\": {true|false}\n }\n }\n\n Where:\n\n :eql:synopsis:``\n Regular statement text.\n\n :eql:synopsis:``\n Statement text template with interpolation points using the ``\\(name)``\n syntax.\n\n :eql:synopsis:``\n The name of an interpolation variable in the statement text template\n for which the user prompt is given.\n\n :eql:synopsis:``\n The text of a user prompt for an interpolation variable.\n\n :eql:synopsis:``\n Prompt for the proposed migration step.\n\n :eql:synopsis:``\n An opaque string identifier for a particular operation prompt.\n The client should not repeat prompts with the same prompt id.\n\n\nCommit migration\n----------------\n\n:eql-statement:\n\nCommit the current migration to the database.\n\n.. eql:synopsis::\n\n commit migration ;\n\nDescription\n^^^^^^^^^^^\n\nThe command ``commit migration`` executes all the commands defined by\nthe current migration and records the migration as the most recent\nmigration in the database.\n\nIssuing ``commit migration`` outside of a migration block initiated\nby :eql:stmt:`start migration` is an error.\n\nExample\n^^^^^^^\n\nCreate and execute the current migration:\n\n.. code-block:: edgeql\n\n commit migration;\n\n\nReset schema to initial\n-----------------------\n\n:eql-statement:\n\nReset the database schema to its initial state.\n\n.. eql:synopsis::\n\n reset schema to initial ;\n\n.. warning::\n\n This command will drop all entities and, as a consequence, all data. You\n won't want to use this statement on a production instance unless you want\n to lose all that instance's data.\n\n\nMigration rewrites DDL commands\n===============================\n\nMigration rewrites allow you to change the migration history as long as your\nfinal schema matches the current database schema.\n\nStart migration rewrite\n-----------------------\n\nStart a migration rewrite.\n\n.. eql:synopsis::\n\n start migration rewrite ;\n\nOnce the migration rewrite is started, you can run any arbitrary DDL until you\nare ready to :ref:`commit ` your new\nmigration history. The most useful DDL in this context will be :ref:`create\nmigration ` statements, which will allow you to\ncreate a sequence of migrations that will become your new migration history.\n\nDeclare savepoint\n-----------------\n\nEstablish a new savepoint within the current migration rewrite.\n\n.. eql:synopsis::\n\n declare savepoint ;\n\nParameters\n^^^^^^^^^^\n\n:eql:synopsis:``\n The name which will be used to identify the new savepoint if you need to\n later release it or roll back to it.\n\nRelease savepoint\n-----------------\n\nDestroys a savepoint previously defined in the current migration rewrite.\n\n.. eql:synopsis::\n\n release savepoint ;\n\nParameters\n^^^^^^^^^^\n\n:eql:synopsis:``\n The name of the savepoint to be released.\n\nRollback to savepoint\n---------------------\n\nRollback to the named savepoint.\n\n.. eql:synopsis::\n\n rollback to savepoint ;\n\nAll changes made after the savepoint are discarded. The savepoint remains valid\nand can be rolled back to again later, if needed.\n\nParameters\n^^^^^^^^^^\n\n:eql:synopsis:``\n The name of the savepoint to roll back to.\n\nRollback\n--------\n\nRollback the entire migration rewrite.\n\n.. eql:synopsis::\n\n rollback ;\n\nAll updates made within the transaction are discarded.\n\n.. _ref_eql_ddl_migrations_rewrites_commit:\n\nCommit migration rewrite\n------------------------\n\nCommit a migration rewrite.\n\n.. eql:synopsis::\n\n commit migration rewrite ;\n" }, { "path": "docs/reference/datamodel/modules.rst", "content": ".. _ref_datamodel_modules:\n.. _ref_eql_sdl_modules:\n\n=======\nModules\n=======\n\nEach |branch| has a schema consisting of several **modules**, each with\na unique name. Modules can be used to organize large schemas into\nlogical units. In practice, though, most users put their entire\nschema inside a single module called ``default``.\n\n.. code-block:: sdl\n\n module default {\n # declare types here\n }\n\n.. _ref_name_resolution:\n\nName resolution\n===============\n\nWhen you define a module that references schema objects from another module,\nyou must use a *fully-qualified* name in the form\n``other_module_name::object_name``:\n\n.. code-block:: sdl\n\n module A {\n type User extending B::AbstractUser;\n }\n\n module B {\n abstract type AbstractUser {\n required name: str;\n }\n }\n\nReserved module names\n=====================\n\nThe following module names are reserved by |Gel| and contain pre-defined\ntypes, utility functions, and operators:\n\n* ``std``: standard types, functions, and operators in the :ref:`standard\n library `\n* ``math``: algebraic and statistical :ref:`functions `\n* ``cal``: local (non-timezone-aware) and relative date/time :ref:`types and\n functions `\n* ``schema``: types describing the :ref:`introspection\n ` schema\n* ``sys``: system-wide entities, such as user roles and\n :ref:`databases `\n* ``cfg``: configuration and settings\n\n\nModules are containers\n======================\n\nThey can contain types, functions, and other modules. Here's an example of an\nempty module:\n\n.. code-block:: sdl\n\n module my_module {}\n\nAnd here's an example of a module with a type:\n\n.. code-block:: sdl\n\n module my_module {\n type User {\n required name: str;\n }\n }\n\n\nNested modules\n==============\n\n.. code-block:: sdl\n\n module dracula {\n type Person {\n required name: str;\n multi places_visited: City;\n strength: int16;\n }\n\n module combat {\n function fight(\n one: dracula::Person,\n two: dracula::Person\n ) -> str\n using (\n (one.name ?? 'Fighter 1') ++ ' wins!'\n IF (one.strength ?? 0) > (two.strength ?? 0)\n ELSE (two.name ?? 'Fighter 2') ++ ' wins!'\n );\n }\n }\n\nYou can chain together module names in a fully-qualified name to traverse a\ntree of nested modules. For example, to call the ``fight`` function in the\nnested module example above, you would use\n``dracula::combat::fight()``.\n\n\nDeclaring modules\n=================\n\nThis section describes the syntax to declare a module in your schema.\n\n\nSyntax\n------\n\n.. sdl:synopsis::\n\n module \"{\"\n [ ]\n ...\n \"}\"\n\nDefine a nested module:\n\n.. sdl:synopsis::\n\n module \"{\"\n [ ]\n module \"{\"\n [ ]\n \"}\"\n ...\n \"}\"\n\n\nDescription\n^^^^^^^^^^^\n\nThe module block declaration defines a new module similar to the\n:eql:stmt:`create module` command, but it also allows putting the\nmodule content as nested declarations:\n\n:sdl:synopsis:``\n Define various schema items that belong to this module.\n\nUnlike :eql:stmt:`create module`, a module block with the\nsame name can appear multiple times in an SDL document. In that case\nall blocks with the same name are merged into a single module under\nthat name. For example:\n\n.. code-block:: sdl\n\n module my_module {\n abstract type Named {\n required name: str;\n }\n }\n\n module my_module {\n type User extending Named;\n }\n\nThe above is equivalent to:\n\n.. code-block:: sdl\n\n module my_module {\n abstract type Named {\n required name: str;\n }\n\n type User extending Named;\n }\n\nTypically, in the documentation examples of SDL the *module block* is\nomitted and instead its contents are described without assuming which\nspecific module they belong to.\n\nIt's also possible to declare modules implicitly. In this style, SDL\ndeclaration uses a :ref:`fully-qualified name ` for the\nitem that is being declared. The *module* part of the *fully-qualified* name\nimplies that a module by that name will be automatically created in the\nschema. The following declaration is equivalent to the previous examples,\nbut it declares module ``my_module`` implicitly:\n\n.. code-block:: sdl\n\n abstract type my_module::Named {\n required name: str;\n }\n\n type my_module::User extending my_module::Named;\n\nA module block can be nested inside another module block to create a nested\nmodule. If you want to reference an entity in a nested module by its\nfully-qualified name, you will need to include all of the containing\nmodules' names: ``::::``\n\n.. _ref_eql_ddl_modules:\n\nDDL commands\n============\n\nThis section describes the low-level DDL commands for creating and dropping\nmodules. You typically don't need to use these commands directly, but\nknowing about them is useful for reviewing migrations.\n\n\nCreate module\n-------------\n\n:eql-statement:\n\nCreate a new module.\n\n.. eql:synopsis::\n\n create module [ :: ] \n [ if not exists ];\n\nThere's a :ref:`corresponding SDL declaration `\nfor a module, although in SDL a module declaration is likely to also\ninclude that module's content.\n\n\nDescription\n^^^^^^^^^^^\n\nThe command ``create module`` defines a new module for the current\n:versionreplace:`database;5.0:branch`. The name of the new module must be\ndistinct from any existing module in the current\n:versionreplace:`database;5.0:branch`. Unlike :ref:`SDL module declaration\n` the ``create module`` command does not have sub-commands;\nmodule contents are created separately.\n\nParameters\n^^^^^^^^^^\n\n:eql:synopsis:`if not exists`\n Normally, creating a module that already exists is an error, but\n with this flag the command will succeed. It is useful for scripts\n that add something to a module or, if the module is missing, the\n module is created as well.\n\nExamples\n^^^^^^^^\n\nCreate a new module:\n\n.. code-block:: edgeql\n\n create module payments;\n\nCreate a new nested module:\n\n.. code-block:: edgeql\n\n create module payments::currencies;\n\n\nDrop module\n-----------\n\n:eql-statement:\n\nRemove a module.\n\n.. eql:synopsis::\n\n drop module ;\n\nDescription\n^^^^^^^^^^^\n\nThe command ``drop module`` removes an existing empty module from the\ncurrent :versionreplace:`database;5.0:branch`. If the module contains any\nschema items, this command will fail.\n\nExamples\n^^^^^^^^\n\nRemove a module:\n\n.. code-block:: edgeql\n\n drop module payments;\n" }, { "path": "docs/reference/datamodel/mutation_rewrites.rst", "content": ".. _ref_datamodel_mutation_rewrites:\n\n=================\nMutation rewrites\n=================\n\n.. index:: modify, modification\n\nMutation rewrites allow you to intercept database mutations (i.e.,\n:ref:`inserts ` and/or :ref:`updates `) and set\nthe value of a property or link to the result of an expression you define. They\ncan be defined in your schema.\n\nMutation rewrites are complementary to :ref:`triggers\n`. While triggers are unable to modify the triggering\nobject, mutation rewrites are built for that purpose.\n\nExample: last modified\n======================\n\nHere's an example of a mutation rewrite that updates a property of a ``Post``\ntype to reflect the time of the most recent modification:\n\n.. code-block:: sdl\n\n type Post {\n required title: str;\n required body: str;\n modified: datetime {\n rewrite insert, update using (datetime_of_statement())\n }\n }\n\nEvery time a ``Post`` is updated, the mutation rewrite will be triggered,\nupdating the ``modified`` property:\n\n.. code-block:: edgeql-repl\n\n db> insert Post {\n ... title := 'One wierd trick to fix all your spelling errors'\n ... };\n {default::Post {id: 19e024dc-d3b5-11ed-968c-37f5d0159e5f}}\n db> select Post {title, modified};\n {\n default::Post {\n title: 'One wierd trick to fix all your spelling errors',\n modified: '2023-04-05T13:23:49.488335Z',\n },\n }\n db> update Post\n ... filter .id = '19e024dc-d3b5-11ed-968c-37f5d0159e5f'\n ... set {title := 'One weird trick to fix all your spelling errors'};\n {default::Post {id: 19e024dc-d3b5-11ed-968c-37f5d0159e5f}}\n db> select Post {title, modified};\n {\n default::Post {\n title: 'One weird trick to fix all your spelling errors',\n modified: '2023-04-05T13:25:04.119641Z',\n },\n }\n\nIn some cases, you will want different rewrites depending on the type of query.\nHere, we will add an ``insert`` rewrite and an ``update`` rewrite:\n\n.. code-block:: sdl\n\n type Post {\n required title: str;\n required body: str;\n created: datetime {\n rewrite insert using (datetime_of_statement())\n }\n modified: datetime {\n rewrite update using (datetime_of_statement())\n }\n }\n\nWith this schema, inserts will set the ``Post`` object's ``created`` property\nwhile updates will set the ``modified`` property:\n\n.. code-block:: edgeql-repl\n\n db> insert Post {\n ... title := 'One wierd trick to fix all your spelling errors'\n ... };\n {default::Post {id: 19e024dc-d3b5-11ed-968c-37f5d0159e5f}}\n db> select Post {title, created, modified};\n {\n default::Post {\n title: 'One wierd trick to fix all your spelling errors',\n created: '2023-04-05T13:23:49.488335Z',\n modified: {},\n },\n }\n db> update Post\n ... filter .id = '19e024dc-d3b5-11ed-968c-37f5d0159e5f'\n ... set {title := 'One weird trick to fix all your spelling errors'};\n {default::Post {id: 19e024dc-d3b5-11ed-968c-37f5d0159e5f}}\n db> select Post {title, created, modified};\n {\n default::Post {\n title: 'One weird trick to fix all your spelling errors',\n created: '2023-04-05T13:23:49.488335Z',\n modified: '2023-04-05T13:25:04.119641Z',\n },\n }\n\n.. note::\n\n Each property may have a single ``insert`` and a single ``update`` mutation\n rewrite rule, or they may have a single rule that covers both.\n\n\nMutation context\n================\n\n.. api-index:: rewrite, __subject__, __specified__, __old__\n\nInside the rewrite rule's expression, you have access to a few special values:\n\n* ``__subject__`` refers to the object type with the new property and link\n values.\n* ``__specified__`` is a named tuple with a key for each property or link in\n the type and a boolean value indicating whether this value was explicitly set\n in the mutation.\n* ``__old__`` refers to the object type with the previous property and link\n values (available for update-only mutation rewrites).\n\nHere are some examples of the special values in use. Maybe your blog hosts\narticles about particularly controversial topics. You could use ``__subject__``\nto enforce a \"cooling off\" period before publishing a blog post:\n\n.. code-block:: sdl\n\n type Post {\n required title: str;\n required body: str;\n publish_time: datetime {\n rewrite insert, update using (\n __subject__.publish_time ?? datetime_of_statement() +\n cal::to_relative_duration(days := 10)\n )\n }\n }\n\nHere we take the post's ``publish_time`` if set or the time the statement is\nexecuted and add 10 days to it. That should give our authors time to consider\nif they want to make any changes before a post goes live.\n\nYou can omit ``__subject__`` in many cases and achieve the same thing:\n\n.. code-block:: sdl-diff\n\n type Post {\n required title: str;\n required body: str;\n publish_time: datetime {\n rewrite insert, update using (\n - __subject__.publish_time ?? datetime_of_statement() +\n + .publish_time ?? datetime_of_statement() +\n cal::to_relative_duration(days := 10)\n )\n }\n }\n\nbut only if the path prefix has not changed. In the following schema, for\nexample, the ``__subject__`` in the rewrite rule is required, because in the\ncontext of the nested ``select`` query, the leading dot resolves from the\n``User`` path:\n\n.. code-block:: sdl\n\n type Post {\n required title: str;\n required body: str;\n author_email: str;\n author_name: str {\n rewrite insert, update using (\n (select User {name} filter .email = __subject__.author_email).name\n )\n }\n }\n type User {\n name: str;\n email: str;\n }\n\n.. note::\n\n Learn more about how this works in our documentation on :ref:`path\n resolution `.\n\nUsing ``__specified__``, we can determine which fields were specified in the\nmutation. This would allow us to track when a single property was last modified\nas in the ``title_modified`` property in this schema:\n\n.. code-block:: sdl\n\n type Post {\n required title: str;\n required body: str;\n title_modified: datetime {\n rewrite update using (\n datetime_of_statement()\n if __specified__.title\n else __old__.title_modified\n )\n }\n }\n\n``__specified__.title`` will be ``true`` if that value was set as part of the\nupdate, and this rewrite mutation rule will update ``title_modified`` to\n``datetime_of_statement()`` in that case.\n\nAnother way you might use this is to set a default value but allow overriding:\n\n.. code-block:: sdl\n\n type Post {\n required title: str;\n required body: str;\n modified: datetime {\n rewrite update using (\n datetime_of_statement()\n if not __specified__.modified\n else .modified\n )\n }\n }\n\nHere, we rewrite ``modified`` on updates to ``datetime_of_statement()`` unless\n``modified`` was set in the update. In that case, we allow the specified value\nto be set. This is different from a :ref:`default\n` value because the rewrite happens on each\nupdate whereas a default value is applied only on insert of a new object.\n\nOne shortcoming in using ``__specified__`` to decide whether to update the\n``modified`` property is that we still don't know whether the value changed ā€”\nonly that it was specified in the query. It's possible the value specified was\nthe same as the existing value. You'd need to check the value itself to decide\nif it has changed.\n\nThis is easy enough for a single value, but what if you want a global\n``modified`` property that is updated only if any of the properties or links\nwere changed? That could get cumbersome quickly for an object of any\ncomplexity.\n\nInstead, you might try casting ``__subject__`` and ``__old__`` to ``json`` and\ncomparing them:\n\n.. code-block:: sdl\n\n type Post {\n required title: str;\n required body: str;\n modified: datetime {\n rewrite update using (\n datetime_of_statement()\n if __subject__ {**} != __old__ {**}\n else __old__.modified\n )\n }\n }\n\nLastly, if we want to add an ``author`` property that can be set for each write\nand keep a history of all the authors, we can do this with the help of\n``__old__``:\n\n.. code-block:: sdl\n\n type Post {\n required title: str;\n required body: str;\n author: str;\n all_authors: array {\n default := >[];\n rewrite update using (\n __old__.all_authors\n ++ [__subject__.author]\n );\n }\n }\n\nOn insert, our ``all_authors`` property will get initialized to an empty array\nof strings. We will rewrite updates to concatenate that array with an array\ncontaining the new author value.\n\n\nCached computed\n===============\n\nMutation rewrites can be used to effectively create a cached computed value as\ndemonstrated with the ``byline`` property in this schema:\n\n.. code-block:: sdl\n\n type Post {\n required title: str;\n required body: str;\n author: str;\n created: datetime {\n rewrite insert using (datetime_of_statement())\n }\n byline: str {\n rewrite insert, update using (\n 'by ' ++\n __subject__.author ++\n ' on ' ++\n to_str(__subject__.created, 'Mon DD, YYYY')\n )\n }\n }\n\nThe ``byline`` property will be updated on each insert or update, but the value\nwill not need to be calculated at read time like a proper :ref:`computed\nproperty `.\n\n\n.. _ref_eql_sdl_mutation_rewrites:\n.. _ref_eql_sdl_mutation_rewrites_syntax:\n\nDeclaring mutation rewrites\n===========================\n\n.. api-index:: rewrite insert, rewrite update, using\n\nThis section describes the syntax to declare mutation rewrites in your schema.\n\nSyntax\n------\n\nDefine a new mutation rewrite corresponding to the :ref:`more explicit DDL\ncommands `.\n\n.. sdl:synopsis::\n\n rewrite {insert | update} [, ...]\n using \n\nMutation rewrites must be defined inside a property or link block.\n\nDescription\n^^^^^^^^^^^\n\nThis declaration defines a new trigger with the following options:\n\n:eql:synopsis:`insert | update [, ...]`\n The query type (or types) the rewrite runs on. Separate multiple values\n with commas to invoke the same rewrite for multiple types of queries.\n\n:eql:synopsis:``\n The expression to be evaluated to produce the new value of the property.\n\n\n.. _ref_eql_ddl_mutation_rewrites:\n\nDDL commands\n============\n\nThis section describes the low-level DDL commands for creatin and\ndropping mutation rewrites. You typically don't need to use these commands\ndirectly, but knowing about them is useful for reviewing migrations.\n\n\nCreate rewrite\n--------------\n\n:eql-statement:\n\nDefine a new mutation rewrite.\n\nWhen creating a new property or link:\n\n.. eql:synopsis::\n\n {create | alter} type \"{\"\n create { property | link } : \"{\"\n create rewrite {insert | update} [, ...]\n using \n \"}\" ;\n \"}\" ;\n\nWhen altering an existing property or link:\n\n.. eql:synopsis::\n\n {create | alter} type \"{\"\n alter { property | link } \"{\"\n create rewrite {insert | update} [, ...]\n using \n \"}\" ;\n \"}\" ;\n\n\nDescription\n^^^^^^^^^^^\n\nThe command ``create rewrite`` nested under ``create type`` or ``alter type``\nand then under ``create property/link`` or ``alter property/link`` defines a\nnew mutation rewrite for the given property or link on the given object.\n\n\nParameters\n^^^^^^^^^^\n\n:eql:synopsis:``\n The name (optionally module-qualified) of the type containing the rewrite.\n\n:eql:synopsis:``\n The name (optionally module-qualified) of the property or link being\n rewritten.\n\n:eql:synopsis:`insert | update [, ...]`\n The query type (or types) that are rewritten. Separate multiple values with\n commas to invoke the same rewrite for multiple types of queries.\n\n\nExamples\n^^^^^^^^\n\nDeclare two mutation rewrites on new properties: one that sets a ``created``\nproperty when a new object is inserted and one that sets a ``modified``\nproperty on each update:\n\n.. code-block:: edgeql\n\n alter type User {\n create property created: datetime {\n create rewrite insert using (datetime_of_statement());\n };\n create property modified: datetime {\n create rewrite update using (datetime_of_statement());\n };\n };\n\n\nDrop rewrite\n------------\n\n:eql-statement:\n\nDrop a mutation rewrite.\n\n.. eql:synopsis::\n\n alter type \"{\"\n alter property \"{\"\n drop rewrite {insert | update} ;\n \"}\" ;\n \"}\" ;\n\n\nDescription\n^^^^^^^^^^^\n\nThe command ``drop rewrite`` inside an ``alter type`` block and further inside\nan ``alter property`` block removes the definition of an existing mutation\nrewrite on the specified property or link of the specified type.\n\n\nParameters\n^^^^^^^^^^\n\n:eql:synopsis:``\n The name (optionally module-qualified) of the type containing the rewrite.\n\n:eql:synopsis:``\n The name (optionally module-qualified) of the property or link being\n rewritten.\n\n:eql:synopsis:`insert | update [, ...]`\n The query type (or types) that are rewritten. Separate multiple values with\n commas to invoke the same rewrite for multiple types of queries.\n\n\nExample\n^^^^^^^\n\nRemove the ``insert`` rewrite of the ``created`` property on the ``User`` type:\n\n.. code-block:: edgeql\n\n alter type User {\n alter property created {\n drop rewrite insert;\n };\n };\n\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`Introspection > Mutation rewrites\n `\n" }, { "path": "docs/reference/datamodel/objects.rst", "content": ".. _ref_datamodel_object_types:\n\n============\nObject Types\n============\n\n.. index:: tables, models\n\n*Object types* are the primary components of a Gel schema. They are\nanalogous to SQL *tables* or ORM *models*, and consist of :ref:`properties\n` and :ref:`links `.\n\nProperties\n==========\n\nProperties are used to attach primitive/scalar data to an object type.\nFor the full documentation on properties, see :ref:`ref_datamodel_props`.\n\n.. code-block:: sdl\n\n type Person {\n email: str;\n }\n\nUsing in a query:\n\n.. code-block:: edgeql\n\n select Person {\n email\n };\n\n\nLinks\n=====\n\nLinks are used to define relationships between object types. For the full\ndocumentation on links, see :ref:`ref_datamodel_links`.\n\n.. code-block:: sdl\n\n type Person {\n email: str;\n best_friend: Person;\n }\n\nUsing in a query:\n\n.. code-block:: edgeql\n\n select Person {\n email,\n best_friend: {\n email\n }\n };\n\nID\n==\n\n.. index:: uuid, primary key\n\nThere's no need to manually declare a primary key on your object types. All\nobject types automatically contain a property ``id`` of type ``UUID`` that's\n*required*, *globally unique*, *readonly*, and has an index on it.\nThe ``id`` is assigned upon creation and cannot be changed.\n\nUsing in a query:\n\n.. code-block:: edgeql\n\n select Person { id };\n select Person { email } filter .id = '123e4567-e89b-...';\n\n\nAbstract types\n==============\n\n.. index:: abstract, inheritance\n\nObject types can either be *abstract* or *non-abstract*. By default all object\ntypes are non-abstract. You can't create or store instances of abstract types\n(a.k.a. mixins), but they're a useful way to share functionality and\nstructure among other object types.\n\n.. code-block:: sdl\n\n abstract type HasName {\n first_name: str;\n last_name: str;\n }\n\n.. _ref_datamodel_objects_inheritance:\n.. _ref_eql_sdl_object_types_inheritance:\n\nInheritance\n===========\n\n.. index:: extending, extends, subtypes, supertypes\n\nObject types can *extend* other object types. The extending type (AKA the\n*subtype*) inherits all links, properties, indexes, constraints, etc. from its\n*supertypes*.\n\n.. code-block:: sdl\n\n abstract type HasName {\n first_name: str;\n last_name: str;\n }\n\n type Person extending HasName {\n email: str;\n best_friend: Person;\n }\n\nUsing in a query:\n\n.. code-block:: edgeql\n\n select Person {\n first_name,\n email,\n best_friend: {\n last_name\n }\n };\n\n\n.. _ref_datamodel_objects_multiple_inheritance:\n\nMultiple Inheritance\n====================\n\nObject types can extend more than one type ā€” that's called\n*multiple inheritance*. This mechanism allows building complex object\ntypes out of combinations of more basic types.\n\n.. note::\n\n Gel's multiple inheritance should not be confused with the multiple\n inheritance of C++ or Python, where the complexity usually arises\n from fine-grained mixing of logic. Gel's multiple inheritance is\n structural and allows for natural composition.\n\n.. code-block:: sdl-diff\n\n abstract type HasName {\n first_name: str;\n last_name: str;\n }\n\n + abstract type HasEmail {\n + email: str;\n + }\n\n - type Person extending HasName {\n + type Person extending HasName, HasEmail {\n - email: str;\n best_friend: Person;\n }\n\nIf multiple supertypes share links or properties, those properties must be\nof the same type and cardinality.\n\n\n.. _ref_eql_sdl_object_types:\n.. _ref_eql_sdl_object_types_syntax:\n\n\nDefining object types\n=====================\n\n.. api-index:: abstract, type, extending\n\nThis section describes the syntax to declare object types in your schema.\n\nSyntax\n------\n\n.. sdl:synopsis::\n\n [abstract] type [extending [, ...] ]\n [ \"{\"\n [ ]\n [ ]\n [ ]\n [ ]\n [ ]\n ...\n \"}\" ]\n\nDescription\n^^^^^^^^^^^\n\nThis declaration defines a new object type with the following options:\n\n:eql:synopsis:`abstract`\n If specified, the created type will be *abstract*.\n\n:eql:synopsis:``\n The name (optionally module-qualified) of the new type.\n\n:eql:synopsis:`extending [, ...]`\n Optional clause specifying the *supertypes* of the new type.\n\n Use of ``extending`` creates a persistent type relationship\n between the new subtype and its supertype(s). Schema modifications\n to the supertype(s) propagate to the subtype.\n\n References to supertypes in queries will also include objects of\n the subtype.\n\n If the same *link* name exists in more than one supertype, or\n is explicitly defined in the subtype and at least one supertype,\n then the data types of the link targets must be *compatible*.\n If there is no conflict, the links are merged to form a single\n link in the new type.\n\nThese sub-declarations are allowed in the ``Type`` block:\n\n:sdl:synopsis:``\n Set object type :ref:`annotation `\n to a given *value*.\n\n:sdl:synopsis:``\n Define a concrete :ref:`property ` for this object type.\n\n:sdl:synopsis:``\n Define a concrete :ref:`link ` for this object type.\n\n:sdl:synopsis:``\n Define a concrete :ref:`constraint ` for this\n object type.\n\n:sdl:synopsis:``\n Define an :ref:`index ` for this object type.\n\n\n.. _ref_eql_ddl_object_types:\n\nDDL commands\n============\n\nThis section describes the low-level DDL commands for creating, altering, and\ndropping object types. You typically don't need to use these commands directly,\nbut knowing about them is useful for reviewing migrations.\n\nCreate type\n-----------\n\n:eql-statement:\n:eql-haswith:\n\nDefine a new object type.\n\n.. eql:synopsis::\n\n [ with [, ...] ]\n create [abstract] type [ extending [, ...] ]\n [ \"{\" ; [...] \"}\" ] ;\n\n # where is one of\n\n create annotation := \n create link ...\n create property ...\n create constraint ...\n create index on \n\nDescription\n^^^^^^^^^^^\n\nThe command ``create type`` defines a new object type for use in the\ncurrent |branch|.\n\nIf *name* is qualified with a module name, then the type is created\nin that module, otherwise it is created in the current module.\nThe type name must be distinct from that of any existing schema item\nin the module.\n\nParameters\n^^^^^^^^^^\n\nMost sub-commands and options of this command are identical to the\n:ref:`SDL object type declaration `,\nwith some additional features listed below:\n\n:eql:synopsis:`with [, ...]`\n Alias declarations.\n\n The ``with`` clause allows specifying module aliases\n that can be referenced by the command. See :ref:`ref_eql_statements_with`\n for more information.\n\nThe following subcommands are allowed in the ``create type`` block:\n\n:eql:synopsis:`create annotation := `\n Set object type :eql:synopsis:`` to\n :eql:synopsis:``.\n\n See :eql:stmt:`create annotation` for details.\n\n:eql:synopsis:`create link ...`\n Define a new link for this object type. See\n :eql:stmt:`create link` for details.\n\n:eql:synopsis:`create property ...`\n Define a new property for this object type. See\n :eql:stmt:`create property` for details.\n\n:eql:synopsis:`create constraint ...`\n Define a concrete constraint for this object type. See\n :eql:stmt:`create constraint` for details.\n\n:eql:synopsis:`create index on `\n Define a new :ref:`index `\n using *index-expr* for this object type. See\n :eql:stmt:`create index` for details.\n\nExample\n^^^^^^^\n\nCreate an object type ``User``:\n\n.. code-block:: edgeql\n\n create type User {\n create property name: str;\n };\n\n\nAlter type\n----------\n\n:eql-statement:\n:eql-haswith:\n\nChange the definition of an object type.\n\n.. eql:synopsis::\n\n [ with [, ...] ]\n alter type \n [ \"{\" ; [...] \"}\" ] ;\n\n [ with [, ...] ]\n alter type ;\n\n # where is one of\n\n rename to \n extending [, ...]\n create annotation := \n alter annotation := \n drop annotation \n create link ...\n alter link ...\n drop link ...\n create property ...\n alter property ...\n drop property ...\n create constraint ...\n alter constraint ...\n drop constraint ...\n create index on \n drop index on \n\nDescription\n^^^^^^^^^^^\n\nThe command ``alter type`` changes the definition of an object type.\n*name* must be a name of an existing object type, optionally qualified\nwith a module name.\n\nParameters\n^^^^^^^^^^\n\n:eql:synopsis:`with [, ...]`\n Alias declarations.\n\n The ``with`` clause allows specifying module aliases\n that can be referenced by the command. See :ref:`ref_eql_statements_with`\n for more information.\n\n:eql:synopsis:``\n The name (optionally module-qualified) of the type being altered.\n\n:eql:synopsis:`extending [, ...]`\n Alter the supertype list. The full syntax of this subcommand is:\n\n .. eql:synopsis::\n\n extending [, ...]\n [ first | last | before | after ]\n\n This subcommand makes the type a subtype of the specified list\n of supertypes. The requirements for the parent-child relationship\n are the same as when creating an object type.\n\n It is possible to specify the position in the parent list\n using the following optional keywords:\n\n * ``first`` -- insert parent(s) at the beginning of the\n parent list,\n * ``last`` -- insert parent(s) at the end of the parent list,\n * ``before `` -- insert parent(s) before an\n existing *parent*,\n * ``after `` -- insert parent(s) after an existing\n *parent*.\n\n:eql:synopsis:`alter annotation ;`\n Alter object type annotation :eql:synopsis:``.\n See :eql:stmt:`alter annotation` for details.\n\n:eql:synopsis:`drop annotation `\n Remove object type :eql:synopsis:``.\n See :eql:stmt:`drop annotation` for details.\n\n:eql:synopsis:`alter link ...`\n Alter the definition of a link for this object type. See\n :eql:stmt:`alter link` for details.\n\n:eql:synopsis:`drop link `\n Remove a link item from this object type. See\n :eql:stmt:`drop link` for details.\n\n:eql:synopsis:`alter property ...`\n Alter the definition of a property item for this object type.\n See :eql:stmt:`alter property` for details.\n\n:eql:synopsis:`drop property `\n Remove a property item from this object type. See\n :eql:stmt:`drop property` for details.\n\n:eql:synopsis:`alter constraint ...`\n Alter the definition of a constraint for this object type. See\n :eql:stmt:`alter constraint` for details.\n\n:eql:synopsis:`drop constraint ;`\n Remove a constraint from this object type. See\n :eql:stmt:`drop constraint` for details.\n\n:eql:synopsis:`drop index on `\n Remove an :ref:`index ` defined as *index-expr*\n from this object type. See :eql:stmt:`drop index` for details.\n\nAll the subcommands allowed in the ``create type`` block are also\nvalid subcommands for the ``alter type`` block.\n\nExample\n^^^^^^^\n\nAlter the ``User`` object type to make ``name`` required:\n\n.. code-block:: edgeql\n\n alter type User {\n alter property name {\n set required;\n }\n };\n\n\nDrop type\n---------\n\n:eql-statement:\n:eql-haswith:\n\nRemove the specified object type from the schema.\n\n.. eql:synopsis::\n\n drop type ;\n\nDescription\n^^^^^^^^^^^\n\nThe command ``drop type`` removes the specified object type from the\nschema. All subordinate schema items defined on this type,\nsuch as links and indexes, are removed as well.\n\nExample\n^^^^^^^\n\nRemove the ``User`` object type:\n\n.. code-block:: edgeql\n\n drop type User;\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`Introspection > Object types\n `\n * - :ref:`Cheatsheets > Object types `\n" }, { "path": "docs/reference/datamodel/permissions.rst", "content": ".. _ref_datamodel_permissions:\n\n.. versionadded:: 7.0\n\n===========\nPermissions\n===========\n\n.. index:: RBAC, role based access control, capability\n\n*Permissions* are the mechanism for limiting access to the database based on\nprovided connection credentials.\n\nEach :ref:`role ` has as set of granted permissions.\n\n.. code-block:: edgeql\n\n create role alice {\n set password := 'wonderland';\n set permissions := {\n sys::perm::data_modifiction,\n default::can_see_secrets\n };\n };\n\nPermissions are either :ref:`built-in ` or\n:ref:`defined in schema `.\n\nSome language features or functions require current role to have certain\npermissions. For example, to use ``insert``, ``update`` or ``delete``, current\nrole is required to have ``sys::perm::data_modification``.\n\nAdditionally, permissions of current role can be accessed via\n:ref:`global variables` of the same name:\n\n.. code-block:: edgeql\n\n select global sys::perm::data_modification;\n\n\nNote that roles are instance-wide object, which means that they exist\nindependent of branches and their schemas. This means that role's permissions\napply to all branches.\n\nRoles that are qualified as *superuser* are implicitly granted\n:ref:`all permissions`.\n\nBuilt-in permissions\n====================\n\n.. _ref_datamodel_permissions_built_in:\n\n:eql:synopsis:`sys::perm::data_modification`\n Required for using ``insert``, ``update`` or ``delete`` statements.\n\n:eql:synopsis:`sys::perm::ddl`\n Required for modification of schema. This includes applying migrations,\n and issuing bare DDL commands (e.g. ``create type Post;``).\n\n It does not include global instance commands, such as ``create branch``\n or ``create role``. These are only allowed to *superuser* roles.\n\n:eql:synopsis:`sys::perm::branch_config`\n Required for issuing ``configure current branch``.\n\n:eql:synopsis:`sys::perm::sql_session_config`\n Required for issuing ``SET`` and ``RESET`` SQL commands.\n\n:eql:synopsis:`sys::perm::analyze`\n Required for issuing ``analyze ...`` queries.\n\n:eql:synopsis:`sys::perm::query_stats_read`\n Required for reading ``sys::QueryStats``.\n\n:eql:synopsis:`sys::perm::approximate_count`\n Required for accessing ``sys::approximate_count()``.\n\n\n:eql:synopsis:`cfg::perm::configure_timeout`\n Required for setting various timeouts, for example\n ``session_idle_transaction_timeout`` and ``query_execution_timeout``.\n\n:eql:synopsis:`cfg::perm::configure_apply_access_policies`\n Required for disabling access policies.\n\n:eql:synopsis:`cfg::perm::configure_allow_user_specified_id`\n Required for setting ``allow_user_specified_id``.\n\n\n:eql:synopsis:`std::net::perm::http_write`\n Required for issuing HTTP requests.\n\n:eql:synopsis:`std::net::perm::http_read`\n Required for reading status of issued HTTP requests and responses.\n\n\nPermissions for :ref:`auth ` extension:\n\n:eql:synopsis:`ext::auth::perm::auth_read`\n\n:eql:synopsis:`ext::auth::perm::auth_write`\n\n:eql:synopsis:`ext::auth::perm::auth_read_user`\n\n\nPermissions for ``ai`` extension are described\nin :ref:`AI extension reference `.\n\n\nCustom permissions\n==================\n\n.. _ref_datamodel_permissions_custom:\n\nCustom permissions can be defined in schema, to fit the security model of each\napplication.\n\n.. code-block:: sql\n\n module default {\n permission data_export;\n }\n\nThese permissions can be assigned to roles, similar to built-in permissions:\n\n.. code-block:: edgeql\n\n alter role warehouse {\n set permissions := {default::data_export};\n };\n\n.. note::\n\n Role permissions are instance-wide.\n\n If an unrelated branch defines ``default::data_export``, the ``warehouse``\n role will receive it as well. This happens even if the unrelated branch\n adds the permission after ``alter role``.\n\n Additionally, a role may be given permissions which do not yet exist in\n any schema. This is useful for creating roles before any schemas are\n applied.\n\n\nTo check if the current database connection's role has a permission, use\n:ref:`global variable` with the same name\nas the permission. This global is a boolean and cannot be manually set.\n\n.. code-block:: edgeql\n\n select global default::data_export;\n\n\nIn combination with access policies, permissions can be used to limit read or\nwrite access of any type:\n\n.. code-block:: sdl\n\n type AuditLog {\n property event: str;\n\n access policy only_export_can_read\n allow select\n using (global data_export);\n\n access policy anyone_can_insert\n allow insert;\n }\n\nIn this example, we have type ``AuditLog`` into which all roles are allowed to\ninsert new log entries. But reading is allowed only to roles that posses\n``data_export`` permission (or are qualified as a *superuser*).\n\n\nCommon patterns\n===============\n\n\nPublic readonly database\n------------------------\n\nGel server can be exposed to public internet, with clients connecting directy\nfrom browsers. Let's assume that only want to grant read access to the public\nbrowser client.\n\nIn such scenarios, it is recommended to create a separate role\nthat will be used by the JavaScript client (e.g. ``webapp``) and not grant it\nany permissions.\n\nThis way, it will not be able to issue ``DROP TYPE`` or ``DELETE`` commands,\nbut will be able to read all data in the database. More importantly, it will\nnot be able to configure ``apply_access_policies`` to ``false`` to bypass\nour restrictions.\n\nIf we want to limit that access further, for example limit read access to type\n``Secrets``, we can use such schema:\n\n.. code-block:: sdl\n\n permission server_access;\n\n type Secret {\n access policy all_access\n allow select, insert, update, delete\n using (global server_access);\n };\n\n\nBecause ``webapp`` role will not possess permission ``server_access`` it will\nnot be able to read (or modify) ``Secret``. For other, trusted clients, which\nshould be able to access ``Secrets``, we have use *superuser* role, or some\nother role with ``server_access`` permission:\n\n.. code-block:: edgeql\n\n create role api_server {\n set password := 'strong_password';\n set permissions := {sys::perm::dml, default::server_access};\n };\n\n\nPublic partially writable database\n----------------------------------\n\nA similar example to the previous one is a public database, with a JavaScript\nclient that needs write access to some, but not all, object types.\n\nIn such scenarios, it is recommended to create a separate role for it\n(e.g. ``webapp``) and assign it ``sys::perm::ddl`` permission.\n\nSuch role will be able to connect to the database, read all data and modify\nall types. For obvious reasons, this is undesirable, since client credentials\ncould be extracted and used to delete all data in the database.\n\nTo further limit access, the access policies must be used on\nevery object:\n\n.. code-block:: sdl\n\n permission server_access;\n\n type Posts { # read-only\n access policy everyone_can_read allow select using (true);\n access policy server_can_do_everything\n allow select, insert, update, delete\n using (global server_access);\n }\n\n type Events { # insert-only\n access policy everyone_can_insert allow insert using (true);\n access policy server_can_do_everything\n allow select, insert, update, delete\n using (global server_access);\n }\n\n type Secrets { # no access\n access policy server_can_do_everything\n allow select, insert, update, delete\n using (global server_access);\n };\n\n\nAgain, we can then use superuser role for server to fully access the database,\nor setup a separate role with ``server_access`` permission.\n\n\nRestricting branches\n--------------------\n\nTo control access by branches instead of by object type, we can use\n``Role.branches`` setting.\n\nFor example, let's assume we have an instance with ``staging`` and ``prod``\nbranches. We want the role ``dev`` to have full access to ``staging``, but not\n``prod``.\n\n.. code-block:: edgeql\n\n create role dev {\n set password := 'strong_password';\n set branches := {'staging'};\n };\n\nFor more about this, see :ref:`Roles `. \n\n\nSuperuser permissions\n=====================\n\n.. _ref_datamodel_permissions_superuser:\n\nRoles with *superuser* status are exempt from permission checks and have full\naccess over the instance.\n\nThis includes some commands that are not covered by any permission and are thus\nallowed *only* to *superuser* roles.\n\nThese commands include:\n\n* :eql:synopsis:`ROLE` commands\n\n* :eql:synopsis:`BRANCH` commands\n\n* :eql:synopsis:`EXTENSION PACKAGE` commands\n\n* :eql:synopsis:`CONFIGURE INSTANCE` command\n\n* :eql:synopsis:`DESCRIBE` command\n\n* :eql:synopsis:`ADMINISTER` command\n\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`Schema > Access policies\n `\n * - :ref:`Running Gel > Administration > Roles `\n\n\n" }, { "path": "docs/reference/datamodel/primitives.rst", "content": ".. _ref_datamodel_primitives:\n\n==========\nPrimitives\n==========\n\n|Gel| has a robust type system consisting of primitive and object types.\ntypes. Primitive types are used to declare *properties* on object types,\nas query and function arguments, as as well as in other contexts.\n\n.. _ref_datamodel_scalars:\n\nBuilt-in scalar types\n=====================\n\nGel comes with a range of built-in scalar types, such as:\n\n* String: :eql:type:`str`\n* Boolean: :eql:type:`bool`\n* Various numeric types: :eql:type:`int16`, :eql:type:`int32`,\n :eql:type:`int64`, :eql:type:`float32`, :eql:type:`float64`, :eql:type:`bigint`, :eql:type:`decimal`\n* JSON: :eql:type:`json`,\n* UUID: :eql:type:`uuid`,\n* Date/time: :eql:type:`datetime`, :eql:type:`duration`\n :eql:type:`cal::local_datetime`, :eql:type:`cal::local_date`,\n :eql:type:`cal::local_time`, :eql:type:`cal::relative_duration`,\n :eql:type:`cal::date_duration`\n* Miscellaneous: :eql:type:`sequence`, :eql:type:`bytes`, etc.\n\nCustom scalars\n==============\n\nYou can extend built-in scalars with additional constraints or annotations.\nHere's an example of a non-negative custom ``int64`` variant:\n\n.. code-block:: sdl\n\n scalar type posint64 extending int64 {\n constraint min_value(0);\n }\n\n.. _ref_datamodel_enums:\n\nEnums\n=====\n\nEnum types are created by extending the abstract :eql:type:`enum` type, e.g.:\n\n.. code-block:: sdl\n\n scalar type Color extending enum;\n\n type Shirt {\n color: Color;\n }\n\nwhich can be queries with:\n\n.. code-block:: edgeql\n\n select Shirt filter .color = Color.Red;\n\nFor a full reference on enum types, see the :ref:`Enum docs `.\n\n.. _ref_datamodel_arrays:\n\nArrays\n======\n\nArrays store zero or more primitive values of the same type in an ordered list.\nArrays cannot contain object types or other arrays, but can contain virtually\nany other type.\n\n.. code-block:: sdl\n\n type Person {\n str_array: array;\n json_array: array;\n tuple_array: array>;\n\n # INVALID: arrays of object types not allowed:\n # friends: array\n\n # INVALID: arrays cannot be nested:\n # nested_array: array>\n\n # VALID: arrays can contain tuples with arrays in them\n nested_array_via_tuple: array>>\n }\n\nArray syntax in EdgeQL is very intuitive (indexing starts at ``0``):\n\n.. code-block:: edgeql\n\n select [1, 2, 3];\n select [1, 2, 3][1] = 2; # true\n\nFor a full reference on array types, see the :ref:`Array docs `.\n\n.. _ref_datamodel_tuples:\n\nTuples\n======\n\nLike arrays, tuples are ordered sequences of primitive data. Unlike arrays,\neach element of a tuple can have a distinct type. Tuple elements can be *any\ntype*, including primitives, objects, arrays, and other tuples.\n\n.. code-block:: sdl\n\n type Person {\n unnamed_tuple: tuple;\n nested_tuple: tuple>>;\n tuple_of_arrays: tuple, array>;\n }\n\nOptionally, you can assign a *key* to each element of the tuple. Tuples\ncontaining explicit keys are known as *named tuples*. You must assign keys to\nall elements (or none of them).\n\n.. code-block:: sdl\n\n type BlogPost {\n metadata: tuple;\n }\n\nNamed and unnamed tuples are the same data structure under the hood. You can\nadd, remove, and change keys in a tuple type after it's been declared. For\ndetails, see :ref:`Tuples `.\n\n.. note::\n\n When you query an *unnamed* tuple using one of EdgeQL's\n :ref:`client libraries `, its value is converted to a\n list/array. When you fetch a named tuple, it is converted into an\n object/dictionary/hashmap depending on the language.\n\n.. _ref_datamodel_ranges:\n\nRanges\n======\n\nRanges represent some interval of values. The intervals can be bound or\nunbound on either end. They can also be empty, containing no values. Only\nsome scalar types have corresponding range types:\n\n- Numeric ranges: ``range``, ``range``, ``range``,\n ``range``, ``range``\n- Date/time ranges: ``range``, ``range``,\n ``range``\n\nExample:\n\n.. code-block:: sdl\n\n type DieRoll {\n values: range;\n }\n\nFor a full reference on ranges, functions and operators see the\n:ref:`Range docs `.\n\nSequences\n=========\n\nTo represent an auto-incrementing integer property, declare a custom scalar\nthat extends the abstract ``sequence`` type. Creating a sequence type\ninitializes a global ``int64`` counter that auto-increments whenever a new\nobject is created. All properties that point to the same sequence type will\nshare the counter.\n\n.. code-block:: sdl\n\n scalar type ticket_number extending sequence;\n type Ticket {\n number: ticket_number;\n rendered_number := 'TICKET-\\(.number)';\n }\n\nFor a full reference on sequences, see the :ref:`Sequence docs `.\n\n.. _ref_eql_sdl_scalars:\n.. _ref_eql_sdl_scalars_syntax:\n\nDeclaring scalars\n=================\n\nThis section describes the syntax to declare a custom scalar type in your\nschema.\n\n\nSyntax\n------\n\n.. sdl:synopsis::\n\n [abstract] scalar type [extending [, ...] ]\n [ \"{\"\n [ ]\n [ ]\n ...\n \"}\" ]\n\nDescription\n^^^^^^^^^^^\n\nThis declaration defines a new object type with the following options:\n\n:eql:synopsis:`abstract`\n If specified, the created scalar type will be *abstract*.\n\n:eql:synopsis:``\n The name (optionally module-qualified) of the new scalar type.\n\n:eql:synopsis:`extending `\n Optional clause specifying the *supertype* of the new type.\n\n If :eql:synopsis:`` is an\n :eql:type:`enumerated type ` declaration then\n an enumerated scalar type is defined.\n\n Use of ``extending`` creates a persistent type relationship\n between the new subtype and its supertype(s). Schema modifications\n to the supertype(s) propagate to the subtype.\n\nThe valid SDL sub-declarations are listed below:\n\n:sdl:synopsis:``\n Set scalar type :ref:`annotation `\n to a given *value*.\n\n:sdl:synopsis:``\n Define a concrete :ref:`constraint ` for\n this scalar type.\n\n\n.. _ref_eql_ddl_scalars:\n\nDDL commands\n============\n\nThis section describes the low-level DDL commands for creating, altering, and\ndropping scalar types. You typically don't need to use these commands directly,\nbut knowing about them is useful for reviewing migrations.\n\nCreate scalar\n-------------\n\n:eql-statement:\n:eql-haswith:\n\nDefine a new scalar type.\n\n.. eql:synopsis::\n\n [ with [, ...] ]\n create [abstract] scalar type [ extending ]\n [ \"{\" ; [...] \"}\" ] ;\n\n # where is one of\n\n create annotation := \n create constraint ...\n\nDescription\n^^^^^^^^^^^\n\nThe command ``create scalar type`` defines a new scalar type for use in the\ncurrent |branch|.\n\nIf *name* is qualified with a module name, then the type is created\nin that module, otherwise it is created in the current module.\nThe type name must be distinct from that of any existing schema item\nin the module.\n\nIf the ``abstract`` keyword is specified, the created type will be\n*abstract*.\n\nAll non-abstract scalar types must have an underlying core\nimplementation. For user-defined scalar types this means that\n``create scalar type`` must have another non-abstract scalar type\nas its *supertype*.\n\nThe most common use of ``create scalar type`` is to define a scalar\nsubtype with constraints.\n\nMost sub-commands and options of this command are identical to the\n:ref:`SDL scalar type declaration `. The\nfollowing subcommands are allowed in the ``create scalar type`` block:\n\n:eql:synopsis:`create annotation := ;`\n Set scalar type's :eql:synopsis:`` to\n :eql:synopsis:``.\n\n See :eql:stmt:`create annotation` for details.\n\n:eql:synopsis:`create constraint ...`\n Define a new constraint for this scalar type. See\n :eql:stmt:`create constraint` for details.\n\n\nExamples\n^^^^^^^^\n\nCreate a new non-negative integer type:\n\n.. code-block:: edgeql\n\n create scalar type posint64 extending int64 {\n create constraint min_value(0);\n };\n\nCreate a new enumerated type:\n\n.. code-block:: edgeql\n\n create scalar type Color\n extending enum;\n\n\nAlter scalar\n------------\n\n:eql-statement:\n:eql-haswith:\n\nAlter the definition of a scalar type.\n\n.. eql:synopsis::\n\n [ with [, ...] ]\n alter scalar type \n \"{\" ; [...] \"}\" ;\n\n # where is one of\n\n rename to \n extending ...\n create annotation := \n alter annotation := \n drop annotation \n create constraint ...\n alter constraint ...\n drop constraint ...\n\nDescription\n^^^^^^^^^^^\n\nThe command ``alter scalar type`` changes the definition of a scalar type.\n*name* must be a name of an existing scalar type, optionally qualified\nwith a module name.\n\nThe following subcommands are allowed in the ``alter scalar type`` block:\n\n:eql:synopsis:`rename to ;`\n Change the name of the scalar type to *newname*.\n\n:eql:synopsis:`extending ...`\n Alter the supertype list. It works the same way as in\n :eql:stmt:`alter type`.\n\n:eql:synopsis:`alter annotation ;`\n Alter scalar type :eql:synopsis:``.\n See :eql:stmt:`alter annotation` for details.\n\n:eql:synopsis:`drop annotation `\n Remove scalar type's :eql:synopsis:`` from\n :eql:synopsis:``.\n See :eql:stmt:`drop annotation` for details.\n\n:eql:synopsis:`alter constraint ...`\n Alter the definition of a constraint for this scalar type. See\n :eql:stmt:`alter constraint` for details.\n\n:eql:synopsis:`drop constraint `\n Remove a constraint from this scalar type. See\n :eql:stmt:`drop constraint` for details.\n\nAll the subcommands allowed in the ``create scalar type`` block are also\nvalid subcommands for ``alter scalar type`` block.\n\n\nExamples\n^^^^^^^^\n\nDefine a new constraint on a scalar type:\n\n.. code-block:: edgeql\n\n alter scalar type posint64 {\n create constraint max_value(100);\n };\n\nAdd one more label to an enumerated type:\n\n.. code-block:: edgeql\n\n alter scalar type Color\n extending enum;\n\n\nDrop scalar\n-----------\n\n:eql-statement:\n:eql-haswith:\n\nRemove a scalar type.\n\n.. eql:synopsis::\n\n [ with [, ...] ]\n drop scalar type ;\n\nDescription\n^^^^^^^^^^^\n\nThe command ``drop scalar type`` removes a scalar type.\n\nParameters\n^^^^^^^^^^\n\n*name*\n The name (optionally qualified with a module name) of an existing\n scalar type.\n\nExample\n^^^^^^^\n\nRemove a scalar type:\n\n.. code-block:: edgeql\n\n drop scalar type posint64;\n" }, { "path": "docs/reference/datamodel/properties.rst", "content": ".. _ref_datamodel_props:\n\n==========\nProperties\n==========\n\n.. index:: property, primitive types, fields, columns\n\nProperties are used to associate primitive data with an :ref:`object type ` or :ref:`link `.\n\n.. code-block:: sdl\n\n type Player {\n property email: str;\n points: int64;\n is_online: bool;\n }\n\nProperties are associated with a *name* (e.g. ``email``) and a primitive\ntype (e.g. ``str``).\n\nThe term *primitive type* is an umbrella term that\nencompasses :ref:`scalar types ` like ``str``,\n:ref:`arrays ` and :ref:`tuples `,\n:ref:`and more `.\n\nProperties can be declared using the ``property`` keyword if that improves\nreadability, or it can be ommitted.\n\n\nRequired properties\n===================\n\n.. index:: not null\n.. api-index:: required, optional\n\nProperties can be either ``optional`` (the default) or ``required``.\n\nE.g. here we have a ``User`` type that's guaranteed to have an ``email``,\nbut ``name`` is optional and can be empty:\n\n.. code-block:: sdl\n\n type User {\n required email: str;\n optional name: str;\n }\n\nSince ``optional`` keyword is the default, we can omit it:\n\n.. code-block:: sdl\n\n type User {\n required email: str;\n name: str;\n }\n\n.. _ref_datamodel_props_cardinality:\n\nCardinality\n===========\n\n.. api-index:: single, multi\n\nProperties have a **cardinality**:\n\n* ``prop: type``, short for ``single prop: type``, can either hold zero or\n one value (that's the default).\n\n* ``multi prop: type`` can hold an *unordered set* of values, which can\n be zero, one, or more values of type ``type``.\n\nFor example:\n\n.. code-block:: sdl\n\n type User {\n\n # \"single\" keyword isn't necessary here:\n # properties are single by default\n single name: str;\n\n # an unordered set of strings\n multi nicknames: str;\n\n # an unordered set of string arrays\n multi set_of_arrays: array;\n }\n\nmulti vs. arrays\n================\n\n``multi`` properties are stored differently than arrays under the hood.\nEssentially they are stored in a separate table ``(owner_id, value)``.\n\n.. rubric:: Pros of multi properties vs. arrays\n\n* ``multi`` properties allow efficient search and mutation of large sets.\n Arrays are much slower for those operations.\n\n* ``multi`` properties can have indexes and constraints appied to\n individual elements; arrays, in general, cannot.\n\n* It's easier to aggregate sets and operate on them than on arrays.\n In many cases arrays would require :ref:`unpacking them into a set\n ` first.\n\n.. rubric:: Cons of multi properties vs. arrays\n\n* On small sets, arrays are faster to retrieve.\n\n* It's easier to retain the original order in arrays. Arrays are ordered,\n but sets are not.\n\n\n.. _ref_datamodel_props_default_values:\n\nDefault values\n==============\n\n.. api-index:: default\n\nProperties can have a default value. This default can be a static value or an\narbitrary EdgeQL expression, which will be evaluated upon insertion.\n\n.. code-block:: sdl\n\n type Player {\n required points: int64 {\n default := 0;\n }\n\n required latitude: float64 {\n default := (360 * random() - 180);\n }\n }\n\n\nReadonly properties\n===================\n\n.. index:: immutable\n.. api-index:: readonly\n\nProperties can be marked as ``readonly``. In the example below, the\n``User.external_id`` property can be set at the time of creation but not\nmodified thereafter.\n\n.. code-block:: sdl\n\n type User {\n required external_id: uuid {\n readonly := true;\n }\n }\n\n\nConstraints\n===========\n\n.. api-index:: constraint\n\nProperties can be augmented wth constraints. The example below showcases a\nsubset of Gel's built-in constraints.\n\n.. code-block:: sdl\n\n type BlogPost {\n title: str {\n constraint exclusive; # all post titles must be unique\n constraint min_len_value(8);\n constraint max_len_value(30);\n constraint regexp(r'^[A-Za-z0-9 ]+$');\n }\n\n status: str {\n constraint one_of('Draft', 'InReview', 'Published');\n }\n\n upvotes: int64 {\n constraint min_value(0);\n constraint max_value(9999);\n }\n }\n\nYou can constrain properties with arbitrary :ref:`EdgeQL ` expressions\nreturning ``bool``. To reference the value of the property, use the special scope\nkeyword ``__subject__``.\n\n.. code-block:: sdl\n\n type BlogPost {\n title: str {\n constraint expression on (\n __subject__ = str_trim(__subject__)\n );\n }\n }\n\nThe constraint above guarantees that ``BlogPost.title`` doesn't contain any\nleading or trailing whitespace by checking that the raw string is equal to the\ntrimmed version. It uses the built-in :eql:func:`str_trim` function.\n\nFor a full reference of built-in constraints, see the :ref:`Constraints\nreference `.\n\n\nAnnotations\n===========\n\n.. index:: metadata\n\nProperties can contain annotations, small human-readable notes. The built-in\nannotations are ``title``, ``description``, and ``deprecated``. You may also\ndeclare :ref:`custom annotation types `.\n\n.. code-block:: sdl\n\n type User {\n email: str {\n annotation title := 'Email address';\n }\n }\n\n\nAbstract properties\n===================\n\n.. api-index:: abstract property\n\nProperties can be *concrete* (the default) or *abstract*. Abstract properties\nare declared independent of a source or target, can contain :ref:`annotations\n`, constraints, indexes, and can be marked as\n``readonly``.\n\n.. code-block:: sdl\n\n abstract property email_prop {\n annotation title := 'An email address';\n readonly := true;\n }\n\n type Student {\n # inherits annotations and \"readonly := true\"\n email: str {\n extending email_prop;\n };\n }\n\nOverloading properties\n======================\n\nAny time we want to amend an inherited property (e.g. to add a constraint),\nthe ``overloaded`` keyword must be used. This is to prevent unintentional\noverloading due to a name clash:\n\n.. code-block:: sdl\n\n abstract type Named {\n optional name: str;\n }\n\n type User extending Named {\n # make \"name\" required\n overloaded required name: str;\n }\n\n\n.. _ref_eql_sdl_props:\n.. _ref_eql_sdl_props_syntax:\n\nDeclaring properties\n====================\n\nSyntax\n------\n\nThis section describes the syntax to declare properties in your schema.\n\n.. sdl:synopsis::\n\n # Concrete property form used inside type declaration:\n [ overloaded ] [{required | optional}] [{single | multi}]\n [ property ] : \n [ \"{\"\n [ extending [, ...] ; ]\n [ default := ; ]\n [ readonly := {true | false} ; ]\n [ ]\n [ ]\n ...\n \"}\" ]\n\n # Computed property form used inside type declaration:\n [{required | optional}] [{single | multi}]\n [ property ] := ;\n\n # Computed property form used inside type declaration (extended):\n [ overloaded ] [{required | optional}] [{single | multi}]\n property [: ]\n [ \"{\"\n using () ;\n [ extending [, ...] ; ]\n [ ]\n [ ]\n ...\n \"}\" ]\n\n # Abstract property form:\n abstract property [::]\n [ \"{\"\n [extending [, ...] ; ]\n [ readonly := {true | false} ; ]\n [ ]\n ...\n \"}\" ]\n\n\nDescription\n^^^^^^^^^^^\n\nThere are several forms of ``property`` declaration, as shown in the\nsyntax synopsis above. The first form is the canonical definition\nform, the second and third forms are used for defining a\n:ref:`computed property `, and the last\none is a form to define an ``abstract property``.\n\nThe abstract form allows declaring the property directly inside\na :ref:`module `.\n\nConcrete property forms are always used as sub-declarations\nfor an :ref:`object type ` or\na :ref:`link `.\n\nThe following options are available:\n\n:eql:synopsis:`overloaded`\n If specified, indicates that the property is inherited and that some\n feature of it may be altered in the current object type. It is an\n error to declare a property as *overloaded* if it is not inherited.\n\n:eql:synopsis:`required`\n If specified, the property is considered *required* for the parent\n object type. It is an error for an object to have a required\n property resolve to an empty value. Child properties **always**\n inherit the *required* attribute, i.e it is not possible to make a\n required property non-required by extending it.\n\n:eql:synopsis:`optional`\n This is the default qualifier assumed when no qualifier is\n specified, but it can also be specified explicitly. The property\n is considered *optional* for the parent object type, i.e. it is\n possible for the property to resolve to an empty value.\n\n:eql:synopsis:`multi`\n Specifies that there may be more than one instance of this\n property in an object, in other words, ``Object.property`` may\n resolve to a set of a size greater than one.\n\n:eql:synopsis:`single`\n Specifies that there may be at most *one* instance of this\n property in an object, in other words, ``Object.property`` may\n resolve to a set of a size not greater than one. ``single`` is\n assumed if nether ``multi`` nor ``single`` qualifier is specified.\n\n:eql:synopsis:`extending [, ...]`\n Optional clause specifying the *parents* of the new property item.\n\n Use of ``extending`` creates a persistent schema relationship\n between the new property and its parents. Schema modifications\n to the parent(s) propagate to the child.\n\n:eql:synopsis:``\n The type must be a valid :ref:`type expression `\n denoting a non-abstract scalar or a container type.\n\nThe valid SDL sub-declarations are listed below:\n\n:eql:synopsis:`default := `\n Specifies the default value for the property as an EdgeQL expression.\n The default value is used in an ``insert`` statement if an explicit\n value for this property is not specified.\n\n The expression must be :ref:`Stable `.\n\n:eql:synopsis:`readonly := {true | false}`\n If ``true``, the property is considered *read-only*.\n Modifications of this property are prohibited once an object is\n created. All of the derived properties **must** preserve the\n original *read-only* value.\n\n:sdl:synopsis:``\n Set property :ref:`annotation `\n to a given *value*.\n\n:sdl:synopsis:``\n Define a concrete :ref:`constraint ` on\n the property.\n\n\n.. _ref_eql_ddl_props:\n\nDDL commands\n============\n\nThis section describes the low-level DDL commands for creating, altering, and\ndropping properties. You typically don't need to use these commands directly,\nbut knowing about them is useful for reviewing migrations.\n\n\n.. _ref_eql_ddl_props_syntax:\n\nCreate property\n---------------\n\n:eql-statement:\n:eql-haswith:\n\nDefine a new property.\n\n.. eql:synopsis::\n\n [ with [, ...] ]\n {create|alter} {type|link} \"{\"\n [ ... ]\n create [{required | optional}] [{single | multi}]\n property \n [ extending [, ...] ] : \n [ \"{\" ; [...] \"}\" ] ;\n [ ... ]\n \"}\"\n\n # Computed property form:\n\n [ with [, ...] ]\n {create|alter} {type|link} \"{\"\n [ ... ]\n create [{required | optional}] [{single | multi}]\n property := ;\n [ ... ]\n \"}\"\n\n # Abstract property form:\n\n [ with [, ...] ]\n create abstract property [::] [extending [, ...]]\n [ \"{\" ; [...] \"}\" ]\n\n # where is one of\n\n set default := \n set readonly := {true | false}\n create annotation := \n create constraint ...\n\nParameters\n^^^^^^^^^^\n\nMost sub-commands and options of this command are identical to the\n:ref:`SDL property declaration `. The\nfollowing subcommands are allowed in the ``create property`` block:\n\n:eql:synopsis:`set default := `\n Specifies the default value for the property as an EdgeQL expression.\n Other than a slight syntactical difference this is the same as the\n corresponding SDL declaration.\n\n:eql:synopsis:`set readonly := {true | false}`\n Specifies whether the property is considered *read-only*. Other\n than a slight syntactical difference this is the same as the\n corresponding SDL declaration.\n\n:eql:synopsis:`create annotation := `\n Set property :eql:synopsis:`` to\n :eql:synopsis:``.\n\n See :eql:stmt:`create annotation` for details.\n\n:eql:synopsis:`create constraint`\n Define a concrete constraint on the property.\n See :eql:stmt:`create constraint` for details.\n\n\nExamples\n^^^^^^^^\n\nDefine a new link ``address`` on the ``User`` object type:\n\n.. code-block:: edgeql\n\n alter type User {\n create property address: str\n };\n\nDefine a new :ref:`computed property `\n``number_of_connections`` on the ``User`` object type counting the\nnumber of interests:\n\n.. code-block:: edgeql\n\n alter type User {\n create property number_of_connections :=\n count(.interests)\n };\n\nDefine a new abstract link ``orderable`` with ``weight`` property:\n\n.. code-block:: edgeql\n\n create abstract link orderable {\n create property weight: std::int64\n };\n\n\nAlter property\n--------------\n\n:eql-statement:\n:eql-haswith:\n\nChange the definition of a property.\n\n.. eql:synopsis::\n\n [ with [, ...] ]\n {create | alter} {type | link} \"{\"\n [ ... ]\n alter property \n [ \"{\" ] ; [...] [ \"}\" ];\n [ ... ]\n \"}\"\n\n\n [ with [, ...] ]\n alter abstract property [::]\n [ \"{\" ] ; [...] [ \"}\" ];\n\n # where is one of\n\n set default := \n reset default\n set readonly := {true | false}\n reset readonly\n rename to \n extending ...\n set required [using ( [using ()\n create annotation := \n alter annotation := \n drop annotation \n create constraint ...\n alter constraint ...\n drop constraint ...\n\n\nParameters\n^^^^^^^^^^\n\n:eql:synopsis:``\n The name of an object type or link on which the property is defined.\n May be optionally qualified with module.\n\n:eql:synopsis:``\n The unqualified name of the property to modify.\n\n:eql:synopsis:``\n Optional name of the module to create or alter the abstract property in.\n If not specified, the current module is used.\n\nThe following subcommands are allowed in the ``alter link`` block:\n\n:eql:synopsis:`rename to `\n Change the name of the property to :eql:synopsis:``.\n All concrete properties inheriting from this property are\n also renamed.\n\n:eql:synopsis:`extending ...`\n Alter the property parent list. The full syntax of this subcommand is:\n\n .. eql:synopsis::\n\n extending [, ...]\n [ first | last | before | after ]\n\n This subcommand makes the property a child of the specified list\n of parent property items. The requirements for the parent-child\n relationship are the same as when creating a property.\n\n It is possible to specify the position in the parent list\n using the following optional keywords:\n\n * ``first`` -- insert parent(s) at the beginning of the\n parent list,\n * ``last`` -- insert parent(s) at the end of the parent list,\n * ``before `` -- insert parent(s) before an\n existing *parent*,\n * ``after `` -- insert parent(s) after an existing\n *parent*.\n\n:eql:synopsis:`set required [using ( [using (`. The optional ``using`` clause specifies\n a conversion expression that computes the new property value from the old.\n The conversion expression must return a singleton set and is evaluated\n on each element of ``multi`` properties. A ``using`` clause must be\n provided if there is no implicit or assignment cast from old to new type.\n\n:eql:synopsis:`reset type`\n Reset the type of the property to the type inherited from properties\n of the same name in supertypes. It is an error to ``reset type`` on\n a property that is not inherited.\n\n:eql:synopsis:`using ()`\n Change the expression of a :ref:`computed property\n `. Only valid for concrete properties.\n\n:eql:synopsis:`alter annotation ;`\n Alter property annotation :eql:synopsis:``.\n See :eql:stmt:`alter annotation` for details.\n\n:eql:synopsis:`drop annotation ;`\n Remove property annotation :eql:synopsis:``.\n See :eql:stmt:`drop annotation` for details.\n\n:eql:synopsis:`alter constraint ...`\n Alter the definition of a constraint for this property. See\n :eql:stmt:`alter constraint` for details.\n\n:eql:synopsis:`drop constraint ;`\n Remove a constraint from this property. See\n :eql:stmt:`drop constraint` for details.\n\n:eql:synopsis:`reset default`\n Remove the default value from this property, or reset it to the value\n inherited from a supertype, if the property is inherited.\n\n:eql:synopsis:`reset readonly`\n Set property writability to the default value (writable), or, if the\n property is inherited, to the value inherited from properties in\n supertypes.\n\nAll the subcommands allowed in the ``create property`` block are also\nvalid subcommands for ``alter property`` block.\n\nExamples\n^^^^^^^^\n\nSet the ``title`` annotation of property ``address`` of object type\n``User`` to ``\"Home address\"``:\n\n.. code-block:: edgeql\n\n alter type User {\n alter property address\n create annotation title := \"Home address\";\n };\n\nAdd a maximum-length constraint to property ``address`` of object type\n``User``:\n\n.. code-block:: edgeql\n\n alter type User {\n alter property address {\n create constraint max_len_value(500);\n };\n };\n\nRename the property ``weight`` of link ``orderable`` to ``sort_by``:\n\n.. code-block:: edgeql\n\n alter abstract link orderable {\n alter property weight rename to sort_by;\n };\n\nRedefine the :ref:`computed property `\n``number_of_connections`` to be the number of friends:\n\n.. code-block:: edgeql\n\n alter type User {\n alter property number_of_connections using (\n count(.friends)\n )\n };\n\n\nDrop property\n-------------\n\n:eql-statement:\n:eql-haswith:\n\nRemove a property from the schema.\n\n.. eql:synopsis::\n\n [ with [, ...] ]\n {create|alter} type \"{\"\n [ ... ]\n drop link \n [ ... ]\n \"}\"\n\n\n [ with [, ...] ]\n drop abstract property ;\n\n\nExample\n^^^^^^^\n\nRemove property ``address`` from type ``User``:\n\n.. code-block:: edgeql\n\n alter type User {\n drop property address;\n };\n" }, { "path": "docs/reference/datamodel/triggers.rst", "content": ".. _ref_datamodel_triggers:\n.. _ref_eql_sdl_triggers:\n\n========\nTriggers\n========\n\nTriggers allow you to define an expression to be executed whenever a given\nquery type is run on an object type. The original query will *trigger* your\npre-defined expression to run in a transaction along with the original query.\nThese can be defined in your schema.\n\n\nImportant notes\n===============\n\nTriggers are an advanced feature and have some caveats that\nyou should be aware of.\n\nConsider using mutation rewrites\n--------------------------------\n\nTriggers cannot be used to *modify* the object that set off the trigger,\nalthough they can be used with :eql:func:`assert` to do *validation* on\nthat object. If you need to modify the object, you can use :ref:`mutation\nrewrites `.\n\nUnified trigger query execution\n-------------------------------\n\nAll queries within triggers, along with the initial triggering query, are\ncompiled into a single combined SQL query under the hood. Keep this in mind\nwhen designing triggers that modify existing records. If multiple ``update``\nqueries within your triggers target the same object, only one of these\nqueries will ultimately be executed. To ensure all desired updates on an\nobject are applied, consolidate them into a single ``update`` query within\none trigger, instead of distributing them across multiple updates.\n\nMulti-stage trigger execution\n-----------------------------\n\nIn some cases, a trigger can cause another trigger to fire. When this\nhappens, Gel completes all the triggers fired by the initial query\nbefore kicking off a new \"stage\" of triggers. In the second stage, any\ntriggers fired by the initial stage of triggers will fire. Gel will\ncontinue adding trigger stages until all triggers are complete.\n\nThe exception to this is when triggers would cause a loop or would cause\nthe same trigger to be run in two different stages. These triggers will\ngenerate an error.\n\nData visibility\n---------------\n\nAny query in your trigger will return the state of the database *after* the\ntriggering query. If this query's results include the object that flipped\nthe trigger, the results will contain that object in the same state as\n``__new__``.\n\n\nExample: audit log\n==================\n\nHere's an example that creates a simple **audit log** type so that we can keep\ntrack of what's happening to our users in a database. First, we will create a\n``Log`` type:\n\n.. code-block:: sdl\n\n type Log {\n action: str;\n timestamp: datetime {\n default := datetime_current();\n }\n target_name: str;\n change: str;\n }\n\nWith the ``Log`` type in place, we can write some triggers that will\nautomatically create ``Log`` objects for any insert, update, or delete queries\non the ``Person`` type:\n\n.. code-block:: sdl\n\n type Person {\n required name: str;\n\n trigger log_insert after insert for each do (\n insert Log {\n action := 'insert',\n target_name := __new__.name\n }\n );\n\n trigger log_update after update for each do (\n insert Log {\n action := 'update',\n target_name := __new__.name,\n change := __old__.name ++ '->' ++ __new__.name\n }\n );\n\n trigger log_delete after delete for each do (\n insert Log {\n action := 'delete',\n target_name := __old__.name\n }\n );\n }\n\nIn a trigger's expression, we have access to the ``__old__`` and/or ``__new__``\nvariables which capture the object before and after the query. Triggers on\n``update`` can use both variables. Triggers on ``delete`` can use ``__old__``.\nTriggers on ``insert`` can use ``__new__``.\n\nNow, whenever we run a query, we get a log entry as well:\n\n.. code-block:: edgeql-repl\n\n db> insert Person {name := 'Jonathan Harker'};\n {default::Person {id: b4d4e7e6-bd19-11ed-8363-1737d8d4c3c3}}\n db> select Log {action, timestamp, target_name, change};\n {\n default::Log {\n action: 'insert',\n timestamp: '2023-03-07T18:56:02.403817Z',\n target_name: 'Jonathan Harker',\n change: {}\n }\n }\n db> update Person filter .name = 'Jonathan Harker'\n ... set {name := 'Mina Murray'};\n {default::Person {id: b4d4e7e6-bd19-11ed-8363-1737d8d4c3c3}}\n db> select Log {action, timestamp, target_name, change};\n {\n default::Log {\n action: 'insert',\n timestamp: '2023-03-07T18:56:02.403817Z',\n target_name: 'Jonathan Harker',\n change: {}\n },\n default::Log {\n action: 'update',\n timestamp: '2023-03-07T18:56:39.520889Z',\n target_name: 'Mina Murray',\n change: 'Jonathan Harker->Mina Murray'\n },\n }\n db> delete Person filter .name = 'Mina Murray';\n {default::Person {id: b4d4e7e6-bd19-11ed-8363-1737d8d4c3c3}}\n db> select Log {action, timestamp, target_name, change};\n {\n default::Log {\n action: 'insert',\n timestamp: '2023-03-07T18:56:02.403817Z',\n target_name: 'Jonathan Harker',\n change: {}\n },\n default::Log {\n action: 'update',\n timestamp: '2023-03-07T18:56:39.520889Z',\n target_name: 'Mina Murray',\n change: 'Jonathan Harker->Mina Murray'\n },\n default::Log {\n action: 'delete',\n timestamp: '2023-03-07T19:00:52.636084Z',\n target_name: 'Mina Murray',\n change: {}\n },\n }\n\nOur audit logging works, but the update logs have a major shortcoming: they\nlog an update even when nothing changes. Any time an ``update`` query runs,\nwe get a log, even if the values are the same. We can prevent that by\nusing the trigger's ``when`` to run the trigger conditionally. Here's a\nrework of our ``update`` logging query:\n\n.. code-block:: sdl-invalid\n\n trigger log_update after update for each\n when (__old__.name != __new__.name)\n do (\n insert Log {\n action := 'update',\n target_name := __new__.name,\n change := __old__.name ++ '->' ++ __new__.name\n }\n );\n\nIf this object were more complicated and we had many properties to compare,\nwe could use a ``json`` cast to compare them all in one shot:\n\n.. code-block:: sdl-invalid\n\n trigger log_update after update for each\n when (__old__ {**} != __new__ {**})\n do (\n insert Log {\n action := 'update',\n target_name := __new__.name,\n change := __old__.name ++ '->' ++ __new__.name\n }\n );\n\nYou might find that one log entry per row is too granular or too noisy for your\nuse case. In that case, a ``for all`` trigger may be a better fit. Here's a\nschema that changes the ``Log`` type so that each object can log multiple\nwrites by making ``target_name`` and ``change`` :ref:`multi properties\n` and switches to ``for all`` triggers:\n\n.. code-block:: sdl-diff\n\n type Log {\n action: str;\n timestamp: datetime {\n default := datetime_current();\n }\n - target_name: str;\n - change: str;\n + multi target_name: str;\n + multi change: str;\n }\n\n type Person {\n required name: str;\n\n - trigger log_insert after insert for each do (\n + trigger log_insert after insert for all do (\n insert Log {\n action := 'insert',\n target_name := __new__.name\n }\n );\n\n - trigger log_update after update for each do (\n + trigger log_update after update for all do (\n insert Log {\n action := 'update',\n target_name := __new__.name,\n change := __old__.name ++ '->' ++ __new__.name\n }\n );\n\n - trigger log_delete after delete for each do (\n + trigger log_delete after delete for all do (\n insert Log {\n action := 'delete',\n target_name := __old__.name\n }\n );\n }\n\nUnder this new schema, each query matching the trigger gets a single ``Log``\nobject instead of one ``Log`` object per row:\n\n.. code-block:: edgeql-repl\n\n db> for name in {'Jonathan Harker', 'Mina Murray', 'Dracula'}\n ... union (\n ... insert Person {name := name}\n ... );\n {\n default::Person {id: 3836f9c8-d393-11ed-9638-3793d3a39133},\n default::Person {id: 38370a8a-d393-11ed-9638-d3e9b92ca408},\n default::Person {id: 38370abc-d393-11ed-9638-5390f3cbd375},\n }\n db> select Log {action, timestamp, target_name, change};\n {\n default::Log {\n action: 'insert',\n timestamp: '2023-03-07T19:12:21.113521Z',\n target_name: {'Jonathan Harker', 'Mina Murray', 'Dracula'},\n change: {},\n },\n }\n db> for change in {\n ... (old_name := 'Jonathan Harker', new_name := 'Jonathan'),\n ... (old_name := 'Mina Murray', new_name := 'Mina')\n ... }\n ... union (\n ... update Person filter .name = change.old_name set {\n ... name := change.new_name\n ... }\n ... );\n {\n default::Person {id: 3836f9c8-d393-11ed-9638-3793d3a39133},\n default::Person {id: 38370a8a-d393-11ed-9638-d3e9b92ca408},\n }\n db> select Log {action, timestamp, target_name, change};\n {\n default::Log {\n action: 'insert',\n timestamp: '2023-04-05T09:21:17.514089Z',\n target_name: {'Jonathan Harker', 'Mina Murray', 'Dracula'},\n change: {},\n },\n default::Log {\n action: 'update',\n timestamp: '2023-04-05T09:35:30.389571Z',\n target_name: {'Jonathan', 'Mina'},\n change: {'Jonathan Harker->Jonathan', 'Mina Murray->Mina'},\n },\n }\n\nExample: validation\n===================\n\n.. index:: trigger, validate, assert\n\nTriggers may also be used for validation by calling :eql:func:`assert` inside\nthe trigger. In this example, the ``Person`` type has two multi links to other\n``Person`` objects named ``friends`` and ``enemies``. These two links should be\nmutually exclusive, so we have written a trigger to make sure there are no\ncommon objects linked in both.\n\n.. code-block:: sdl\n\n type Person {\n required name: str;\n multi friends: Person;\n multi enemies: Person;\n\n trigger prohibit_frenemies after insert, update for each do (\n assert(\n not exists (__new__.friends intersect __new__.enemies),\n message := \"Invalid frenemies\",\n )\n )\n }\n\nWith this trigger in place, it is impossible to link the same ``Person`` as\nboth a friend and an enemy of any other person.\n\n.. code-block:: edgeql-repl\n\n db> insert Person {name := 'Quincey Morris'};\n {default::Person {id: e4a55480-d2de-11ed-93bd-9f4224fc73af}}\n db> insert Person {name := 'Dracula'};\n {default::Person {id: e7f2cff0-d2de-11ed-93bd-279780478afb}}\n db> update Person\n ... filter .name = 'Quincey Morris'\n ... set {\n ... enemies := (\n ... select detached Person filter .name = 'Dracula'\n ... )\n ... };\n {default::Person {id: e4a55480-d2de-11ed-93bd-9f4224fc73af}}\n db> update Person\n ... filter .name = 'Quincey Morris'\n ... set {\n ... friends := (\n ... select detached Person filter .name = 'Dracula'\n ... )\n ... };\n gel error: GelError: Invalid frenemies\n\n\nExample: logging\n================\n\nDeclare a trigger that inserts a ``Log`` object for each new ``User`` object:\n\n.. code-block:: sdl\n\n type User {\n required name: str;\n\n trigger log_insert after insert for each do (\n insert Log {\n action := 'insert',\n target_name := __new__.name\n }\n );\n }\n\nDeclare a trigger that inserts a ``Log`` object conditionally when an update\nquery makes a change to a ``User`` object:\n\n.. code-block:: sdl\n\n type User {\n required name: str;\n\n trigger log_update after update for each\n when (__old__ {**} != __new__ {**})\n do (\n insert Log {\n action := 'update',\n target_name := __new__.name,\n change := __old__.name ++ '->' ++ __new__.name\n }\n );\n }\n\n\n.. _ref_eql_sdl_triggers_syntax:\n\n\nDeclaring triggers\n==================\n\n.. api-index:: trigger, after insert, after update, after delete, for each,\n for all, when, do, __new__, __old__\n\nThis section describes the syntax to declare a trigger in your schema.\n\nSyntax\n------\n\n.. sdl:synopsis::\n\n type \"{\"\n trigger \n after\n {insert | update | delete} [, ...]\n for {each | all}\n [ when () ]\n do \n \"}\"\n\nDescription\n-----------\n\nThis declaration defines a new trigger with the following options:\n\n:eql:synopsis:``\n The name (optionally module-qualified) of the type to be triggered on.\n\n:eql:synopsis:``\n The name of the trigger.\n\n:eql:synopsis:`insert | update | delete [, ...]`\n The query type (or types) to trigger on. Separate multiple values with\n commas to invoke the same trigger for multiple types of queries.\n\n:eql:synopsis:`each`\n The expression will be evaluated once per modified object. ``__new__`` and\n ``__old__`` in this context within the expression will refer to a single\n object.\n\n:eql:synopsis:`all`\n The expression will be evaluted once for the entire query, even if multiple\n objects were modified. ``__new__`` and ``__old__`` in this context within\n the expression refer to sets of the modified objects.\n\n.. versionadded:: 4.0\n\n :eql:synopsis:`when ()`\n Optionally provide a condition for the trigger. If the condition is\n met, the trigger will run. If not, the trigger is skipped.\n\n:eql:synopsis:``\n The expression to be evaluated when the trigger is invoked.\n\nThe trigger name must be distinct from that of any existing trigger\non the same type.\n\n\n.. _ref_eql_ddl_triggers:\n\nDDL commands\n============\n\nThis section describes the low-level DDL commands for creating and dropping\ntriggers. You typically don't need to use these commands directly, but\nknowing about them is useful for reviewing migrations.\n\n\nCreate trigger\n--------------\n\n:eql-statement:\n\n:ref:`Define ` a new trigger.\n\n.. eql:synopsis::\n\n {create | alter} type \"{\"\n create trigger \n after\n {insert | update | delete} [, ...]\n for {each | all}\n [ when () ]\n do \n \"}\"\n\nDescription\n^^^^^^^^^^^\n\nThe command ``create trigger`` nested under ``create type`` or ``alter type``\ndefines a new trigger for a given object type.\n\nThe trigger name must be distinct from that of any existing trigger\non the same type.\n\nParameters\n^^^^^^^^^^\n\nThe options of this command are identical to the\n:ref:`SDL trigger declaration `.\n\nExample\n^^^^^^^\n\nDeclare a trigger that inserts a ``Log`` object for each new ``User`` object:\n\n.. code-block:: edgeql\n\n alter type User {\n create trigger log_insert after insert for each do (\n insert Log {\n action := 'insert',\n target_name := __new__.name\n }\n );\n };\n\n.. versionadded:: 4.0\n\n Declare a trigger that inserts a ``Log`` object conditionally when an update\n query makes a change to a ``User`` object:\n\n .. code-block:: edgeql\n\n alter type User {\n create trigger log_update after update for each\n when (__old__ {**} != __new__ {**})\n do (\n insert Log {\n action := 'update',\n target_name := __new__.name,\n change := __old__.name ++ '->' ++ __new__.name\n }\n );\n }\n\nDrop trigger\n------------\n\n:eql-statement:\n\nRemove a trigger.\n\n.. eql:synopsis::\n\n alter type \"{\"\n drop trigger ;\n \"}\"\n\nDescription\n^^^^^^^^^^^\n\nThe command ``drop trigger`` inside an ``alter type`` block removes the\ndefinition of an existing trigger on the specified type.\n\nParameters\n^^^^^^^^^^\n\n:eql:synopsis:``\n The name (optionally module-qualified) of the type being triggered on.\n\n:eql:synopsis:``\n The name of the trigger.\n\nExample\n^^^^^^^\n\nRemove the ``log_insert`` trigger on the ``User`` type:\n\n.. code-block:: edgeql\n\n alter type User {\n drop trigger log_insert;\n };\n\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`Introspection > Triggers `\n" }, { "path": "docs/reference/edgeql/analyze.rst", "content": ".. _ref_eql_analyze:\n\nAnalyze\n=======\n\n.. index:: explain, performance, postgres query planner\n.. api-index:: analyze\n\nPrefix an EdgeQL query with ``analyze`` to run a performance analysis of that\nquery.\n\n.. code-block:: edgeql-repl\n\n db> analyze select Hero {\n ... name,\n ... secret_identity,\n ... villains: {\n ... name,\n ... nemesis: {\n ... name\n ... }\n ... }\n ... };\n ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ Query ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€\n analyze select āžŠ Hero {name, secret_identity, āž‹ villains: {name, āžŒ nemesis: {name}}};\n\n ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ Coarse-grained Query Plan ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€\n ā”‚ Time Cost Loops Rows Width ā”‚ Relations\n āžŠ root ā”‚ 0.0 69709.48 1.0 0.0 32 ā”‚ Hero\n ā•°ā”€ā”€āž‹ .villains ā”‚ 0.0 92.9 0.0 0.0 32 ā”‚ Villain, Hero.villains\n ā•°ā”€ā”€āžŒ .nemesis ā”‚ 0.0 8.18 0.0 0.0 32 ā”‚ Hero\n\n\n.. note::\n\n In addition to using the ``analyze`` statement in the CLI or UI's REPL, you\n may also run performance analysis via our CLI's :ref:`analyze command\n ` and the UI's query builder (accessible by running\n :ref:`ref_cli_gel_ui` to invoke your instance's UI) by prepending your\n query with ``analyze``. This method offers helpful visualizations to to\n make it easy to understand your query's performance.\n\nAfter analyzing a query, you may run the ``\\expand`` command in the REPL to see\nmore fine-grained performance metrics on the previously analyzed query.\n\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`CLI > gel analyze `\n * - :ref:`Reference > EdgeQL > analyze `\n" }, { "path": "docs/reference/edgeql/delete.rst", "content": ".. _ref_eql_delete:\n\nDelete\n======\n\n.. api-index:: delete\n\nThe ``delete`` command is used to delete objects from the database.\n\n.. code-block:: edgeql\n\n delete Hero\n filter .name = 'Iron Man';\n\nClauses\n-------\n\nDeletion statements support ``filter``, ``order by``, ``offset``, and ``limit``\nclauses. See :ref:`EdgeQL > Select ` for full documentation\non these clauses.\n\n.. code-block:: edgeql\n\n delete Hero\n filter .name ilike 'the %'\n order by .name\n offset 10\n limit 5;\n\nLink deletion\n-------------\n\n.. api-index:: ConstraintViolationError\n\nEvery link is associated with a *link deletion policy*. By default, it isn't\npossible to delete an object linked to by another.\n\n.. code-block:: edgeql-repl\n\n db> delete Hero filter .name = \"Yelena Belova\";\n ConstraintViolationError: deletion of default::Hero\n (af7076e0-3e98-11ec-abb3-b3435bbe7c7e) is prohibited by link target policy\n {}\n\nThis deletion failed because Yelena is still in the ``characters`` list of\nthe Black Widow movie. We must destroy this link before Yelena can be\ndeleted.\n\n.. code-block:: edgeql-repl\n\n db> update Movie\n ... filter .title = \"Black Widow\"\n ... set {\n ... characters -= (select Hero filter .name = \"Yelena Belova\")\n ... };\n {default::Movie {id: af706c7c-3e98-11ec-abb3-4bbf3f18a61a}}\n db> delete Hero filter .name = \"Yelena Belova\";\n {default::Hero {id: af7076e0-3e98-11ec-abb3-b3435bbe7c7e}}\n\nTo avoid this behavior, we could update the ``Movie.characters`` link to use\nthe ``allow`` deletion policy.\n\n.. code-block:: sdl-diff\n\n type Movie {\n required title: str { constraint exclusive };\n required release_year: int64;\n - multi characters: Person;\n + multi characters: Person {\n + on target delete allow;\n + };\n }\n\n\nCascading deletes\n^^^^^^^^^^^^^^^^^\n\n.. index:: deletion policy\n.. api-index:: delete source, delete target\n\nIf a link uses the ``delete source`` policy, then deleting a *target* of the\nlink will also delete the object that links to it (the *source*). This behavior\ncan be used to implement cascading deletes; be careful with this power!\n\nThe full list of deletion policies is documented at :ref:`Schema > Links\n`.\n\nReturn value\n------------\n\n.. index:: returning\n\nA ``delete`` statement returns the set of deleted objects. You can pass this\nset into ``select`` to fetch properties and links of the (now-deleted)\nobjects. This is the last moment this data will be available before being\npermanently deleted.\n\n.. code-block:: edgeql-repl\n\n db> with movie := (delete Movie filter .title = \"Untitled\")\n ... select movie {id, title};\n {default::Movie {\n id: b11303c6-40ac-11ec-a77d-d393cdedde83,\n title: 'Untitled',\n }}\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`Reference > Commands > Delete `\n * - :ref:`Cheatsheets > Deleting data `\n" }, { "path": "docs/reference/edgeql/for.rst", "content": ".. _ref_eql_for:\n\nFor\n===\n\n.. api-index:: for in, union\n\nEdgeQL supports a top-level ``for`` statement. These \"for loops\" iterate over\neach element of some input set, execute some expression with it, and merge the\nresults into a single output set.\n\n.. code-block:: edgeql-repl\n\n db> for number in {0, 1, 2, 3}\n ... union (\n ... select { number, number + 0.5 }\n ... );\n {0, 0.5, 1, 1.5, 2, 2.5, 3, 3.5}\n\nThis statement iterates through each number in the set. Inside the loop, the\n``number`` variable is bound to a singleton set. The inner expression is\nexecuted for every element of the input set, and the results of each execution\nare merged into a single output set.\n\n.. note::\n\n The ``union`` keyword is required prior to |EdgeDB| 5.0 and is intended to\n indicate explicitly that the results of each loop execution are ultimately\n merged.\n\n.. versionadded: 5.0\n\n If the body of ``for`` is a statement ā€” ``select``, ``insert``, ``update``,\n ``delete``, ``group``, or ``with`` ā€” ``union`` and the parentheses\n surrounding the statement are no longer required:\n\n .. code-block:: edgeql-repl\n\n db> for number in {0, 1, 2, 3}\n ... select { number, number + 0.5 }\n {0, 0.5, 1, 1.5, 2, 2.5, 3, 3.5}\n\n\nBulk inserts\n------------\n\nThe ``for`` statement is commonly used for bulk inserts.\n\n.. code-block:: edgeql-repl\n\n db> for hero_name in {'Cersi', 'Ikaris', 'Thena'}\n ... union (\n ... insert Hero { name := hero_name }\n ... );\n {\n default::Hero {id: d7d7e0f6-40ae-11ec-87b1-3f06bed494b9},\n default::Hero {id: d7d7f870-40ae-11ec-87b1-f712a4efc3a5},\n default::Hero {id: d7d7f8c0-40ae-11ec-87b1-6b8685d56610}\n }\n\nThis statement iterates through each name in the list of names. Inside the\nloop, ``hero_name`` is bound to a ``str`` singleton, so it can be assigned to\n``Hero.name``.\n\nInstead of literal sets, it's common to use a :ref:`json `\nparameter for bulk inserts. This value is then \"unpacked\" into a set of\n``json`` elements and used inside the ``for`` loop:\n\n.. code-block:: edgeql-repl\n\n db> with\n ... raw_data := $data,\n ... for item in json_array_unpack(raw_data) union (\n ... insert Hero { name := item['name'] }\n ... );\n Parameter $data: [{\"name\":\"Sersi\"},{\"name\":\"Ikaris\"},{\"name\":\"Thena\"}]\n {\n default::Hero {id: d7d7e0f6-40ae-11ec-87b1-3f06bed494b9},\n default::Hero {id: d7d7f870-40ae-11ec-87b1-f712a4efc3a5},\n default::Hero {id: d7d7f8c0-40ae-11ec-87b1-6b8685d56610}\n }\n\n\nA similar approach can be used for bulk updates.\n\n\n.. _ref_eql_for_conditional_dml:\n\nConditional DML\n---------------\n\n.. api-index:: for, if else, unless conflict\n\n.. versionadded:: 4.0\n\n DML is now supported in ``if..else``.\n\nDML (i.e., :ref:`insert `, :ref:`update `,\n:ref:`delete `) is not supported in :eql:op:`if..else`. If you\nneed to do one of these conditionally, you can use a ``for`` loop as a\nworkaround. For example, you might want to write this conditional:\n\n.. code-block::\n\n # šŸš« Does not work\n with admin := (select User filter .role = 'admin')\n select admin if exists admin\n else (insert User {role := 'admin'});\n\nBecause of the lack of support for DML in a conditional, this query will fail.\nHere's how you can accomplish the same thing using the workaround:\n\n.. code-block:: edgeql\n\n # āœ… Works!\n with\n admin := (select User filter .role = 'admin'),\n new := (for _ in (select () filter not exists admin) union (\n insert User {role := 'admin'}\n )),\n select {admin, new};\n\nThe ``admin`` alias represents the condition we want to test for. In this case,\n\"do we have a ``User`` object with a value of ``admin`` for the ``role``\nproperty?\" In the ``new`` alias, we write a ``for`` loop with a ``select``\nquery that will produce a set with a single value if that object we queried for\ndoes *not* exist. (You can use ``exists`` instead of ``not exists`` in the\nnested ``select`` inside the ``for`` loop if you don't want to invert the\ncondition.)\n\nA set with a single value results in a single iteration of the ``for`` loop.\nInside that loop, we run our conditional DML ā€” in this case to insert an admin\nuser. Then we ``select`` both aliases to execute both of their queries. The\nquery will return the ``User`` object. This in effect gives us a query that\nwill insert a ``User`` object with a ``role`` of ``admin`` if none exists or\nreturn that object if it *does* exist.\n\n.. note::\n\n If you're trying to conditionally run DML in response to a violation of an\n exclusivity constraint, you don't need this workaround. You should use\n :ref:`unless conflict ` instead.\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`Reference > Commands > For `\n" }, { "path": "docs/reference/edgeql/group.rst", "content": ".. _ref_eql_group:\n\nGroup\n=====\n\n.. index:: analytics, aggregate\n.. api-index:: group by, group using by, key, grouping, elements, rollup, cube\n\nEdgeQL supports a top-level ``group`` statement. This is used to partition\nsets into subsets based on some parameters. These subsets then can be\nadditionally aggregated to provide some analytics.\n\nThe most basic format is just using the bare :eql:stmt:`group` to group a set\nof objects by some property:\n\n.. code-block:: edgeql-repl\n\n db> group Movie by .release_year;\n {\n {\n key: {release_year: 2016},\n grouping: {'release_year'},\n elements: {\n default::Movie {title: 'Captain America: Civil War'},\n default::Movie {title: 'Doctor Strange'},\n },\n },\n {\n key: {release_year: 2017},\n grouping: {'release_year'},\n elements: {\n default::Movie {title: 'Spider-Man: Homecoming'},\n default::Movie {title: 'Thor: Ragnarok'},\n },\n },\n {\n key: {release_year: 2018},\n grouping: {'release_year'},\n elements: {default::Movie {title: 'Ant-Man and the Wasp'}},\n },\n {\n key: {release_year: 2019},\n grouping: {'release_year'},\n elements: {default::Movie {title: 'Spider-Man: No Way Home'}},\n },\n {\n key: {release_year: 2021},\n grouping: {'release_year'},\n elements: {default::Movie {title: 'Black Widow'}},\n },\n ...\n }\n\nNotice that the result of ``group`` is a set of :ref:`free objects\n` with three fields:\n\n* ``key``: another free object containing the specific value of the\n grouping parameter for a given subset.\n* ``grouping``: set of names of grouping parameters, i.e. the specific\n names that also appear in the ``key`` free object.\n* ``elements``: the actual subset of values that match the ``key``.\n\nIn the ``group`` statement, referring to the property in the ``by`` clause\n**must** be done by using the leading dot shothand ``.release_year``. The\nproperty name then shows up in ``grouping`` and ``key`` to indicate the\ndefining characteristics of the particular result. Alternatively, we can give\nit an alias in an optional ``using`` clause and then that alias can be used in\nthe ``by`` clause and will appear in the results:\n\n.. code-block:: edgeql-repl\n\n db> group Movie {title}\n ... using year := .release_year by year;\n {\n {\n key: {year: 2016},\n grouping: {'year'},\n elements: {\n default::Movie {title: 'Captain America: Civil War'},\n default::Movie {title: 'Doctor Strange'},\n },\n },\n {\n key: {year: 2017},\n grouping: {'year'},\n elements: {\n default::Movie {title: 'Spider-Man: Homecoming'},\n default::Movie {title: 'Thor: Ragnarok'},\n },\n },\n {\n key: {year: 2018},\n grouping: {'year'},\n elements: {default::Movie {title: 'Ant-Man and the Wasp'}},\n },\n {\n key: {year: 2019},\n grouping: {'year'},\n elements: {default::Movie {title: 'Spider-Man: No Way Home'}},\n },\n {\n key: {year: 2021},\n grouping: {'year'},\n elements: {default::Movie {title: 'Black Widow'}},\n },\n ...\n }\n\nThe ``using`` clause is perfect for defining a more complex expression to\ngroup things by. For example, instead of grouping by the ``release_year`` we\ncan group by the release decade:\n\n.. code-block:: edgeql-repl\n\n db> group Movie {title}\n ... using decade := .release_year // 10\n ... by decade;\n {\n {\n {\n key: {decade: 200},\n grouping: {'decade'},\n elements: {\n default::Movie {title: 'Spider-Man'},\n default::Movie {title: 'Spider-Man 2'},\n default::Movie {title: 'Spider-Man 3'},\n default::Movie {title: 'Iron Man'},\n default::Movie {title: 'The Incredible Hulk'},\n },\n },\n {\n key: {decade: 201},\n grouping: {'decade'},\n elements: {\n default::Movie {title: 'Iron Man 2'},\n default::Movie {title: 'Thor'},\n default::Movie {title: 'Captain America: The First Avenger'},\n default::Movie {title: 'The Avengers'},\n default::Movie {title: 'Iron Man 3'},\n default::Movie {title: 'Thor: The Dark World'},\n default::Movie {title: 'Captain America: The Winter Soldier'},\n default::Movie {title: 'Ant-Man'},\n default::Movie {title: 'Captain America: Civil War'},\n default::Movie {title: 'Doctor Strange'},\n default::Movie {title: 'Spider-Man: Homecoming'},\n default::Movie {title: 'Thor: Ragnarok'},\n default::Movie {title: 'Ant-Man and the Wasp'},\n default::Movie {title: 'Spider-Man: No Way Home'},\n },\n },\n {\n key: {decade: 202},\n grouping: {'decade'},\n elements: {default::Movie {title: 'Black Widow'}},\n },\n }\n\nIt's also possible to group by more than one parameter, so we can group by\nwhether the movie ``title`` contains a colon *and* the decade it was released.\nAdditionally, let's only consider more recent movies, say, released after\n2015, so that we're not overwhelmed by all the combination of results:\n\n.. code-block:: edgeql-repl\n\n db> with\n ... # Apply the group query only to more recent movies\n ... M := (select Movie filter .release_year > 2015)\n ... group M {title}\n ... using\n ... decade := .release_year // 10,\n ... has_colon := .title like '%:%'\n ... by decade, has_colon;\n {\n {\n key: {decade: 201, has_colon: false},\n grouping: {'decade', 'has_colon'},\n elements: {\n default::Movie {title: 'Ant-Man and the Wasp'},\n default::Movie {title: 'Doctor Strange'},\n },\n },\n {\n key: {decade: 201, has_colon: true},\n grouping: {'decade', 'has_colon'},\n elements: {\n default::Movie {title: 'Captain America: Civil War'},\n default::Movie {title: 'Spider-Man: No Way Home'},\n default::Movie {title: 'Thor: Ragnarok'},\n default::Movie {title: 'Spider-Man: Homecoming'},\n },\n },\n {\n key: {decade: 202, has_colon: false},\n grouping: {'decade', 'has_colon'},\n elements: {default::Movie {title: 'Black Widow'}},\n },\n }\n\nOnce we break a set into partitions, we can also use :ref:`aggregate\n` functions to provide some analytics about the data.\nFor example, for the above partitioning (by decade and presence of ``:`` in\nthe ``title``) we can calculate how many movies are in each subset as well as\nthe average number of words in the movie titles:\n\n.. code-block:: edgeql-repl\n\n db> with\n ... # Apply the group query only to more recent movies\n ... M := (select Movie filter .release_year > 2015),\n ... groups := (\n ... group M {title}\n ... using\n ... decade := .release_year // 10 - 200,\n ... has_colon := .title like '%:%'\n ... by decade, has_colon\n ... )\n ... select groups {\n ... key := .key {decade, has_colon},\n ... count := count(.elements),\n ... avg_words := math::mean(\n ... len(str_split(.elements.title, ' ')))\n ... };\n {\n {key: {decade: 1, has_colon: false}, count: 2, avg_words: 3},\n {key: {decade: 1, has_colon: true}, count: 4, avg_words: 3},\n {key: {decade: 2, has_colon: false}, count: 1, avg_words: 2},\n }\n\n.. note::\n\n It is possible to produce results that are grouped in multiple different\n ways using :ref:`grouping sets `. This may be\n useful in more sophisticated analytics.\n\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`Reference > Commands > Group `\n" }, { "path": "docs/reference/edgeql/index.rst", "content": ".. versioned-section::\n\n.. _ref_edgeql:\n\n======\nEdgeQL\n======\n\n.. toctree::\n :maxdepth: 3\n :hidden:\n\n literals\n sets\n paths\n types\n parameters\n select\n insert\n update\n delete\n for\n group\n with\n analyze\n path_resolution\n transactions\n\nEdgeQL is a next-generation query language designed to match SQL in power and\nsurpass it in terms of clarity, brevity, and intuitiveness. It's used to query\nthe database, insert/update/delete data, modify/introspect the schema, manage\ntransactions, and more.\n\nDesign goals\n------------\n\nEdgeQL is a spiritual successor to SQL designed with a few core principles in\nmind.\n\n**Compatible with modern languages**. A jaw-dropping amount of effort has been\nspent attempting to `bridge the gap `_ between the *relational*\nparadigm of SQL and the *object-oriented* nature of modern programming\nlanguages. Gel sidesteps this problem by modeling data in an\n*object-relational* way.\n\n**Strongly typed**. EdgeQL is *inextricably tied* to Gel's rigorous\nobject-oriented type system. The type of all expressions is statically\ninferred by Gel.\n\n**Designed for programmers**. EdgeQL prioritizes syntax over keywords; It uses\n``{ curly braces }`` to define scopes/structures and the *assignment\noperator* ``:=`` to set values. The result is a query language that looks more\nlike code and less like word soup.\n\n.. All told, EdgeQL syntax contains roughly 180\n.. reserved keywords; by comparison Postgres-flavored SQL contains `469\n.. `_.\n\n.. **Compiles to SQL**. All EdgeQL queries, no matter how complex, compile to a\n.. single PostgreSQL query under the hood. With the exception of ``group by``,\n.. EdgeQL is equivalent to SQL in terms of power and expressivity.\n\n**Easy deep querying**. Gel's object-relational nature makes it painless\nto write deep, performant queries that traverse links, no ``JOINs`` required.\n\n**Composable**. `Unlike SQL\n`_,\nEdgeQL's syntax is readily composable; queries can be cleanly nested without\nworrying about Cartesian explosion.\n" }, { "path": "docs/reference/edgeql/insert.rst", "content": ".. _ref_eql_insert:\n\nInsert\n======\n\n.. api-index:: insert, :=\n\nThe ``insert`` command is used to create instances of object types. The code\nsamples on this page assume the following schema:\n\n.. code-block:: sdl\n\n module default {\n abstract type Person {\n required name: str { constraint exclusive };\n }\n\n type Hero extending Person {\n secret_identity: str;\n multi villains := . insert Hero {\n ... name := \"Spider-Man\",\n ... secret_identity := \"Peter Parker\"\n ... };\n {default::Hero {id: b0fbe9de-3e90-11ec-8c12-ffa2d5f0176a}}\n\nSimilar to :ref:`selecting fields ` in ``select``, ``insert``\nstatements include a *shape* specified with ``curly braces``; the values of\nproperties/links are assigned with the ``:=`` operator.\n\nOptional links or properties can be omitted entirely, as well as those with a\n``default`` value (like ``id``).\n\n.. code-block:: edgeql-repl\n\n db> insert Hero {\n ... name := \"Spider-Man\"\n ... # secret_identity is omitted\n ... };\n {default::Hero {id: b0fbe9de-3e90-11ec-8c12-ffa2d5f0176a}}\n\nYou can only ``insert`` instances of concrete (non-abstract) object types.\n\n.. code-block:: edgeql-repl\n\n db> insert Person {\n ... name := \"The Man With No Name\"\n ... };\n error: QueryError: cannot insert into abstract object type 'default::Person'\n\nBy default, ``insert`` returns only the inserted object's ``id`` as seen in the\nexamples above. If you want to get additional data back, you may wrap your\n``insert`` with a ``select`` and apply a shape specifying any properties and\nlinks you want returned:\n\n.. code-block:: edgeql-repl\n\n db> select (insert Hero {\n ... name := \"Spider-Man\"\n ... # secret_identity is omitted\n ... }) {id, name};\n {\n default::Hero {\n id: b0fbe9de-3e90-11ec-8c12-ffa2d5f0176a,\n name: \"Spider-Man\"\n }\n }\n\nYou can use :ref:`ref_eql_with` to tidy this up if you prefer:\n\n.. code-block:: edgeql-repl\n\n db> with NewHero := (insert Hero {\n ... name := \"Spider-Man\"\n ... # secret_identity is omitted\n ... })\n ... select NewHero {\n ... id,\n ... name,\n ... }\n {\n default::Hero {\n id: b0fbe9de-3e90-11ec-8c12-ffa2d5f0176a,\n name: \"Spider-Man\"\n }\n }\n\n\n.. _ref_eql_insert_links:\n\nInserting links\n---------------\n\nEdgeQL's composable syntax makes link insertion painless. Below, we insert\n\"Spider-Man: No Way Home\" and include all known heroes and villains as\n``characters`` (which is basically true).\n\n.. code-block:: edgeql-repl\n\n db> insert Movie {\n ... title := \"Spider-Man: No Way Home\",\n ... release_year := 2021,\n ... characters := (\n ... select Person\n ... filter .name in {\n ... 'Spider-Man',\n ... 'Doctor Strange',\n ... 'Doc Ock',\n ... 'Green Goblin'\n ... }\n ... )\n ... };\n {default::Movie {id: 9b1cf9e6-3e95-11ec-95a2-138eeb32759c}}\n\nTo assign to the ``Movie.characters`` link, we're using a *subquery*. This\nsubquery is executed and resolves to a set of type ``Person``, which is\nassignable to ``characters``. Note that the inner ``select Person`` statement\nis wrapped in parentheses; this is required for all subqueries in EdgeQL.\n\nNow let's assign to a *single link*.\n\n.. code-block:: edgeql-repl\n\n db> insert Villain {\n ... name := \"Doc Ock\",\n ... nemesis := (select Hero filter .name = \"Spider-Man\")\n ... };\n\n\nThis query is valid because the inner subquery is guaranteed to return at most\none ``Hero`` object, due to the uniqueness constraint on ``Hero.name``. If you\nare filtering on a non-exclusive property, use ``assert_single`` to guarantee\nthat the subquery will return zero or one results. If more than one result is\nreturned, this query will fail at runtime.\n\n.. code-block:: edgeql-repl\n\n db> insert Villain {\n ... name := \"Doc Ock\",\n ... nemesis := assert_single((\n ... select Hero\n ... filter .secret_identity = \"Peter B. Parker\"\n ... ))\n ... };\n\n\n.. _ref_eql_insert_nested:\n\nNested inserts\n--------------\n\nJust as we used subqueries to populate links with existing objects, we can also\nexecute *nested inserts*.\n\n.. code-block:: edgeql-repl\n\n db> insert Villain {\n ... name := \"The Mandarin\",\n ... nemesis := (insert Hero {\n ... name := \"Shang-Chi\",\n ... secret_identity := \"Shaun\"\n ... })\n ... };\n {default::Villain {id: d47888a0-3e7b-11ec-af13-fb68c8777851}}\n\n\nNow let's write a nested insert for a ``multi`` link.\n\n.. code-block:: edgeql-repl\n\n db> insert Movie {\n ... title := \"Black Widow\",\n ... release_year := 2021,\n ... characters := {\n ... (select Hero filter .name = \"Black Widow\"),\n ... (insert Hero { name := \"Yelena Belova\"}),\n ... (insert Villain {\n ... name := \"Dreykov\",\n ... nemesis := (select Hero filter .name = \"Black Widow\")\n ... })\n ... }\n ... };\n {default::Movie {id: af706c7c-3e98-11ec-abb3-4bbf3f18a61a}}\n\nWe are using :ref:`set literal syntax ` to construct a\nset literal containing several ``select`` and ``insert`` subqueries. This set\ncontains a mix of ``Hero`` and ``Villain`` objects; since these are both\nsubtypes of ``Person`` (the expected type of ``Movie.characters``), this is\nvalid.\n\nYou also can't *assign* to a computed property or link; these fields don't\nactually exist in the database.\n\n.. code-block:: edgeql-repl\n\n db> insert Hero {\n ... name := \"Ant-Man\",\n ... villains := (select Villain)\n ... };\n error: QueryError: modification of computed link 'villains' of object type\n 'default::Hero' is prohibited\n\n.. _ref_eql_insert_with:\n\nWith block\n----------\n\n.. api-index:: with\n\nIn the previous query, we selected Black Widow twice: once in the\n``characters`` set and again as the ``nemesis`` of Dreykov. In circumstances\nlike this, pulling a subquery into a ``with`` block lets you avoid\nduplication.\n\n.. code-block:: edgeql-repl\n\n db> with black_widow := (select Hero filter .name = \"Black Widow\")\n ... insert Movie {\n ... title := \"Black Widow\",\n ... release_year := 2021,\n ... characters := {\n ... black_widow,\n ... (insert Hero { name := \"Yelena Belova\"}),\n ... (insert Villain {\n ... name := \"Dreykov\",\n ... nemesis := black_widow\n ... })\n ... }\n ... };\n {default::Movie {id: af706c7c-3e98-11ec-abb3-4bbf3f18a61a}}\n\n\nThe ``with`` block can contain an arbitrary number of clauses; later clauses\ncan reference earlier ones.\n\n.. code-block:: edgeql-repl\n\n db> with\n ... black_widow := (select Hero filter .name = \"Black Widow\"),\n ... yelena := (insert Hero { name := \"Yelena Belova\"}),\n ... dreykov := (insert Villain {name := \"Dreykov\", nemesis := black_widow})\n ... insert Movie {\n ... title := \"Black Widow\",\n ... release_year := 2021,\n ... characters := { black_widow, yelena, dreykov }\n ... };\n {default::Movie {id: af706c7c-3e98-11ec-abb3-4bbf3f18a61a}}\n\n\n.. _ref_eql_insert_conflicts:\n\nConflicts\n---------\n\n.. api-index:: unless conflict on, else\n\n|Gel| provides a general-purpose mechanism for gracefully handling possible\nexclusivity constraint violations. Consider a scenario where we are trying to\n``insert`` Eternals (the ``Movie``), but we can't remember if it already exists\nin the database.\n\n.. code-block:: edgeql-repl\n\n db> insert Movie {\n ... title := \"Eternals\",\n ... release_year := 2021\n ... }\n ... unless conflict on .title\n ... else (select Movie);\n {default::Movie {id: af706c7c-3e98-11ec-abb3-4bbf3f18a61a}}\n\nThis query attempts to ``insert`` Eternals. If it already exists in the\ndatabase, it will violate the uniqueness constraint on ``Movie.title``, causing\na *conflict* on the ``title`` field. The ``else`` clause is then executed and\nreturned instead. In essence, ``unless conflict`` lets us \"catch\" exclusivity\nconflicts and provide a fallback expression.\n\n.. note::\n\n Note that the ``else`` clause is simply ``select Movie``. There's no need to\n apply additional filters on ``Movie``; in the context of the ``else`` clause,\n ``Movie`` is bound to the conflicting object.\n\n.. note::\n\n Using ``unless conflict`` on :ref:`multi properties\n ` is only supported in 2.10 and later.\n\n.. _ref_eql_upsert:\n\nUpserts\n^^^^^^^\n\nThere are no limitations on what the ``else`` clause can contain; it can be any\nEdgeQL expression, including an :ref:`update ` statement. This\nlets you express *upsert* logic in a single EdgeQL query.\n\n.. code-block:: edgeql-repl\n\n db> with\n ... title := \"Eternals\",\n ... release_year := 2021\n ... insert Movie {\n ... title := title,\n ... release_year := release_year\n ... }\n ... unless conflict on .title\n ... else (\n ... update Movie set { release_year := release_year }\n ... );\n {default::Movie {id: f1bf5ac0-3e9d-11ec-b78d-c7dfb363362c}}\n\nWhen a conflict occurs during the initial ``insert``, the statement falls back\nto the ``update`` statement in the ``else`` clause. This updates the\n``release_year`` of the conflicting object.\n\n.. note::\n\n It can be useful to know the outcome of an upsert. Here's an example\n showing how you can return that:\n\n .. code-block:: edgeql-repl\n\n db> with\n ... title := \"Eternals\",\n ... release_year := 2021,\n ... movie := (\n ... insert Movie {\n ... title := title,\n ... release_year := release_year\n ... }\n ... unless conflict on .title\n ... else (\n ... update Movie set { release_year := release_year }\n ... )\n ... )\n ... select movie {\n ... is_new := (movie not in Movie)\n ... };\n {default::Movie {is_new: true}}\n\n This technique exploits the fact that a ``select`` will not return an\n object inserted in the same query. We know that, if the record exists, we\n updated it. If it does not, we inserted it.\n\n By wrapping your upsert in a ``select`` and putting a shape on it that\n queries for the object and returns whether or not it exists (as ``is_new``,\n in this example), you can easily see whether the object was inserted or\n updated.\n\n If you want to also return some of the ``Movie`` object's data, drop\n additional property names into the shape alongside ``is_new``. If you're on\n 3.0+, you can add ``Movie.*`` to the shape alongside ``is_new`` to get back\n all of the ``Movie`` object's properties. You could even silo the data off,\n keeping it separate from the ``is_new`` computed value like this:\n\n .. code-block:: edgeql-repl\n\n db> with\n ... title := \"Eternals\",\n ... release_year := 2021,\n ... movie := (\n ... insert Movie {\n ... title := title,\n ... release_year := release_year\n ... }\n ... unless conflict on .title\n ... else (\n ... update Movie set { release_year := release_year }\n ... )\n ... )\n ... select {\n ... data := (select movie {*}),\n ... is_new := (movie not in Movie)\n ... };\n {\n {\n data: {\n default::Movie {\n id: 6880d0ba-62ca-11ee-9608-635818746433,\n release_year: 2021,\n title: 'Eternals'\n }\n },\n is_new: false\n }\n }\n\n\nSuppressing failures\n^^^^^^^^^^^^^^^^^^^^\n\n.. api-index:: unless conflict\n\nThe ``else`` clause is optional; when omitted, the ``insert`` statement will\nreturn an *empty set* if a conflict occurs. This is a common way to prevent\n``insert`` queries from failing on constraint violations.\n\n.. code-block:: edgeql-repl\n\n db> insert Hero { name := \"The Wasp\" } # initial insert\n ... unless conflict;\n {default::Hero {id: 35b97a92-3e9b-11ec-8e39-6b9695d671ba}}\n db> insert Hero { name := \"The Wasp\" } # The Wasp now exists\n ... unless conflict;\n {}\n\n.. _ref_eql_insert_bulk:\n\nBulk inserts\n------------\n\nBulk inserts are performed by passing in a JSON array as a :ref:`query\nparameter `, :eql:func:`unpacking ` it, and\nusing a :ref:`for loop ` to insert the objects.\n\n.. code-block:: edgeql-repl\n\n db> with\n ... raw_data := $data,\n ... for item in json_array_unpack(raw_data) union (\n ... insert Hero { name := item['name'] }\n ... );\n Parameter $data: [{\"name\":\"Sersi\"},{\"name\":\"Ikaris\"},{\"name\":\"Thena\"}]\n {\n default::Hero {id: 35b97a92-3e9b-11ec-8e39-6b9695d671ba},\n default::Hero {id: 35b97a92-3e9b-11ec-8e39-6b9695d671ba},\n default::Hero {id: 35b97a92-3e9b-11ec-8e39-6b9695d671ba},\n ...\n }\n\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`Reference > Commands > Insert `\n * - :ref:`Cheatsheets > Inserting data `\n" }, { "path": "docs/reference/edgeql/literals.rst", "content": ".. _ref_eql_literals:\n\nLiterals\n========\n\n.. index:: primitive types\n\nEdgeQL is *inextricably tied* to Gel's rigorous type system. Below is an\noverview of how to declare a literal value of each *primitive type*. Click a\nlink in the left column to jump to the associated section.\n\n.. list-table::\n\n * - :ref:`String `\n - ``str``\n\n * - :ref:`Boolean `\n - ``bool``\n\n * - :ref:`Numbers `\n - ``int16`` ``int32`` ``int64``\n ``float32`` ``float64`` ``bigint``\n ``decimal``\n\n * - :ref:`UUID `\n - ``uuid``\n\n * - :ref:`Enums `\n - ``enum``\n\n * - :ref:`Dates and times `\n - ``datetime`` ``duration``\n ``cal::local_datetime`` ``cal::local_date``\n ``cal::local_time`` ``cal::relative_duration``\n\n * - :ref:`Durations `\n - ``duration`` ``cal::relative_duration`` ``cal::date_duration``\n\n * - :ref:`Ranges `\n - ``range``\n\n * - :ref:`Bytes `\n - ``bytes``\n\n * - :ref:`Arrays `\n - ``array``\n\n * - :ref:`Tuples `\n - ``tuple`` or\n ``tuple``\n\n * - :ref:`JSON `\n - ``json``\n\n.. _ref_eql_literal_strings:\n\nStrings\n-------\n\n.. index:: unicode, quotes, raw strings, escape character\n.. api-index:: str, r'', r\"\", $$, $Ā§labelĀ§$, \\\\Ā§charĀ§\n\nThe :eql:type:`str` type is a variable-length string of Unicode characters. A\nstring can be declared with either single or double quotes.\n\n.. code-block:: edgeql-repl\n\n db> select 'I ā¤ļø EdgeQL';\n {'I ā¤ļø EdgeQL'}\n db> select \"hello there!\";\n {'hello there!'}\n db> select 'hello\\nthere!';\n {'hello\n there!'}\n db> select 'hello\n ... there!';\n {'hello\n there!'}\n db> select r'hello\n ... there!'; # multiline\n {'hello\n there!'}\n\nThere is a special syntax for declaring \"raw strings\". Raw strings treat the\nbackslash ``\\`` as a literal character instead of an escape character.\n\n.. code-block:: edgeql-repl\n\n db> select r'hello\\nthere'; # raw string\n {r'hello\\\\nthere'}\n db> select $$one\n ... two\n ... three$$; # multiline raw string\n {'one\n two\n three'}\n db> select $label$You can add an interstitial label\n ... if you need to use \"$$\" in your string.$label$;\n {\n 'You can add an interstital label\n if you need to use \"$$\" in your string.',\n }\n\nEdgeQL contains a set of built-in functions and operators for searching,\ncomparing, and manipulating strings.\n\n.. code-block:: edgeql-repl\n\n db> select 'hellothere'[5:10];\n {'there'}\n db> select 'hello' ++ 'there';\n {'hellothere'}\n db> select len('hellothere');\n {10}\n db> select str_trim(' hello there ');\n {'hello there'}\n db> select str_split('hello there', ' ');\n {['hello', 'there']}\n\n\nFor a complete reference on strings, see :ref:`Standard Library > String\n` or click an item below.\n\n.. list-table::\n\n * - Indexing and slicing\n - :eql:op:`str[i] ` :eql:op:`str[from:to] `\n * - Concatenation\n - :eql:op:`str ++ str `\n * - Utilities\n - :eql:func:`len`\n * - Transformation functions\n - :eql:func:`str_split` :eql:func:`str_lower` :eql:func:`str_upper`\n :eql:func:`str_title` :eql:func:`str_pad_start` :eql:func:`str_pad_end`\n :eql:func:`str_trim` :eql:func:`str_trim_start` :eql:func:`str_trim_end`\n :eql:func:`str_repeat`\n * - Comparison operators\n - :eql:op:`= ` :eql:op:`\\!= ` :eql:op:`?= `\n :eql:op:`?!= ` :eql:op:`\\< ` :eql:op:`\\> `\n :eql:op:`\\<= ` :eql:op:`\\>= `\n * - Search\n - :eql:func:`contains` :eql:func:`find`\n * - Pattern matching and regexes\n - :eql:op:`str like pattern ` :eql:op:`str ilike pattern `\n :eql:func:`re_match` :eql:func:`re_match_all` :eql:func:`re_replace`\n :eql:func:`re_test`\n\n\n.. _ref_eql_literal_boolean:\n\nBooleans\n--------\n\n.. api-index:: bool\n\nThe :eql:type:`bool` type represents a true/false value.\n\n.. code-block:: edgeql-repl\n\n db> select true;\n {true}\n db> select false;\n {false}\n\n|Gel| provides a set of operators that operate on boolean values.\n\n.. list-table::\n\n * - Comparison operators\n - :eql:op:`= ` :eql:op:`\\!= ` :eql:op:`?= `\n :eql:op:`?!= ` :eql:op:`\\< ` :eql:op:`\\> `\n :eql:op:`\\<= ` :eql:op:`\\>= `\n * - Logical operators\n - :eql:op:`or` :eql:op:`and` :eql:op:`not`\n * - Aggregation\n - :eql:func:`all` :eql:func:`any`\n\n\n.. _ref_eql_literal_numbers:\n\nNumbers\n-------\n\nThere are several numerical types in Gel's type system.\n\n.. list-table::\n\n * - :eql:type:`int16`\n - 16-bit integer\n\n * - :eql:type:`int32`\n - 32-bit integer\n\n * - :eql:type:`int64`\n - 64-bit integer\n\n * - :eql:type:`float32`\n - 32-bit floating point number\n\n * - :eql:type:`float64`\n - 64-bit floating point number\n\n * - :eql:type:`bigint`\n - Arbitrary precision integer.\n\n * - :eql:type:`decimal`\n - Arbitrary precision number.\n\nNumber literals that *do not* contain a decimal are interpreted as ``int64``.\nNumbers containing decimals are interpreted as ``float64``. The ``n`` suffix\ndesignates a number with *arbitrary precision*: either ``bigint`` or\n``decimal``.\n\n====================================== =============================\n Syntax Inferred type\n====================================== =============================\n :eql:code:`select 3;` :eql:type:`int64`\n :eql:code:`select 3.14;` :eql:type:`float64`\n :eql:code:`select 314e-2;` :eql:type:`float64`\n :eql:code:`select 42n;` :eql:type:`bigint`\n :eql:code:`select 42.0n;` :eql:type:`decimal`\n :eql:code:`select 42e+100n;` :eql:type:`decimal`\n\n====================================== =============================\n\nTo declare an ``int16``, ``int32``, or ``float32``, you must provide an\nexplicit type cast. For details on type casting, see :ref:`Casting\n`.\n\n====================================== =============================\n Syntax Type\n====================================== =============================\n :eql:code:`select 1234;` :eql:type:`int16`\n :eql:code:`select 123456;` :eql:type:`int32`\n :eql:code:`select 123.456;` :eql:type:`float32`\n====================================== =============================\n\nEdgeQL includes a full set of arithmetic and comparison operators. Parentheses\ncan be used to indicate the order-of-operations or visually group\nsubexpressions; this is true across all EdgeQL queries.\n\n.. code-block:: edgeql-repl\n\n db> select 5 > 2;\n {true}\n db> select 2 + 2;\n {4}\n db> select 2 ^ 10;\n {1024}\n db> select (1 + 1) * 2 / (3 + 8);\n {0.36363636363636365}\n\n\nEdgeQL provides a comprehensive set of built-in functions and operators on\nnumerical data.\n\n.. list-table::\n\n * - Comparison operators\n - :eql:op:`= ` :eql:op:`\\!= ` :eql:op:`?= `\n :eql:op:`?!= ` :eql:op:`\\< ` :eql:op:`\\> `\n :eql:op:`\\<= ` :eql:op:`\\>= `\n * - Arithmetic\n - :eql:op:`+ ` :eql:op:`- ` :eql:op:`- `\n :eql:op:`* ` :eql:op:`/
` :eql:op:`// `\n :eql:op:`% ` :eql:op:`^ `\n * - Statistics\n - :eql:func:`sum` :eql:func:`min` :eql:func:`max` :eql:func:`math::mean`\n :eql:func:`math::stddev` :eql:func:`math::stddev_pop`\n :eql:func:`math::var` :eql:func:`math::var_pop`\n * - Math\n - :eql:func:`round` :eql:func:`math::abs` :eql:func:`math::ceil`\n :eql:func:`math::floor` :eql:func:`math::ln` :eql:func:`math::lg`\n :eql:func:`math::log`\n * - Random number\n - :eql:func:`random`\n\n\n.. _ref_eql_literal_uuid:\n\nUUID\n----\n\nThe :eql:type:`uuid` type is commonly used to represent object identifiers.\nUUID literal must be explicitly cast from a string value matching the UUID\nspecification.\n\n.. code-block:: edgeql-repl\n\n db> select 'a5ea6360-75bd-4c20-b69c-8f317b0d2857';\n {a5ea6360-75bd-4c20-b69c-8f317b0d2857}\n\nGenerate a random UUID.\n\n.. code-blocK:: edgeql-repl\n\n db> select uuid_generate_v1mc();\n {b4d94e6c-3845-11ec-b0f4-93e867a589e7}\n\n\n.. _ref_eql_literal_enum:\n\nEnums\n-----\n\n.. api-index:: scalar type, extending enum\n\nEnum types must be :ref:`declared in your schema `.\n\n.. code-block:: sdl\n\n scalar type Color extending enum;\n\nOnce declared, an enum literal can be declared with dot notation, or by\ncasting an appropriate string literal:\n\n.. code-block:: edgeql-repl\n\n db> select Color.Red;\n {Red}\n db> select \"Red\";\n {Red}\n\n\n.. _ref_eql_literal_dates:\n\nDates and times\n---------------\n\n|Gel's| typesystem contains several temporal types.\n\n.. list-table::\n\n * - :eql:type:`datetime`\n - Timezone-aware point in time\n\n * - :eql:type:`cal::local_datetime`\n - Date and time w/o timezone\n\n * - :eql:type:`cal::local_date`\n - Date type\n\n * - :eql:type:`cal::local_time`\n - Time type\n\nAll temporal literals are declared by casting an appropriately formatted\nstring.\n\n.. code-block:: edgeql-repl\n\n db> select '1999-03-31T15:17:00Z';\n {'1999-03-31T15:17:00Z'}\n db> select '1999-03-31T17:17:00+02';\n {'1999-03-31T15:17:00Z'}\n db> select '1999-03-31T15:17:00';\n {'1999-03-31T15:17:00'}\n db> select '1999-03-31';\n {'1999-03-31'}\n db> select '15:17:00';\n {'15:17:00'}\n\nEdgeQL supports a set of functions and operators on datetime types.\n\n.. list-table::\n\n * - Comparison operators\n - :eql:op:`= ` :eql:op:`\\!= ` :eql:op:`?= `\n :eql:op:`?!= ` :eql:op:`\\< ` :eql:op:`\\> `\n :eql:op:`\\<= ` :eql:op:`\\>= `\n * - Arithmetic\n - :eql:op:`dt + dt ` :eql:op:`dt - dt `\n * - String parsing\n - :eql:func:`to_datetime` :eql:func:`cal::to_local_datetime`\n :eql:func:`cal::to_local_date` :eql:func:`cal::to_local_time`\n * - Component extraction\n - :eql:func:`datetime_get` :eql:func:`cal::time_get`\n :eql:func:`cal::date_get`\n * - Truncation\n - :eql:func:`datetime_truncate`\n * - System timestamps\n - :eql:func:`datetime_current` :eql:func:`datetime_of_transaction`\n :eql:func:`datetime_of_statement`\n\n\n.. _ref_eql_literal_durations:\n\nDurations\n---------\n\n|Gel's| type system contains three duration types.\n\n\n.. list-table::\n\n * - :eql:type:`duration`\n - Exact duration\n * - :eql:type:`cal::relative_duration`\n - Duration in relative units\n * - :eql:type:`cal::date_duration`\n - Duration in months and days only\n\nExact durations\n^^^^^^^^^^^^^^^\n\nThe :eql:type:`duration` type represents *exact* durations that can be\nrepresented by some fixed number of microseconds. It can be negative and it\nsupports units of ``microseconds``, ``milliseconds``, ``seconds``, ``minutes``,\nand ``hours``.\n\n.. code-block:: edgeql-repl\n\n db> select '45.6 seconds';\n {'0:00:45.6'}\n db> select '-15 microseconds';\n {'-0:00:00.000015'}\n db> select '5 hours 4 minutes 3 seconds';\n {'5:04:03'}\n db> select '8760 hours'; # about a year\n {'8760:00:00'}\n\nAll temporal units beyond ``hour`` no longer correspond to a fixed duration of\ntime; the length of a day/month/year/etc changes based on daylight savings\ntime, the month in question, leap years, etc.\n\nRelative durations\n^^^^^^^^^^^^^^^^^^\n\nBy contrast, the :eql:type:`cal::relative_duration` type represents a\n\"calendar\" duration, like ``1 month``. Because months have different number of\ndays, ``1 month`` doesn't correspond to a fixed number of milliseconds, but\nit's often a useful quantity to represent recurring events, postponements, etc.\n\n.. note::\n\n The ``cal::relative_duration`` type supports the same units as ``duration``,\n plus ``days``, ``weeks``, ``months``, ``years``, ``decades``, ``centuries``,\n and ``millennia``.\n\nTo declare relative duration literals:\n\n.. code-block:: edgeql-repl\n\n db> select '15 milliseconds';\n {'PT.015S'}\n db> select '2 months 3 weeks 45 minutes';\n {'P2M21DT45M'}\n db> select '-7 millennia';\n {'P-7000Y'}\n\nDate durations\n^^^^^^^^^^^^^^\n\nThe :eql:type:`cal::date_duration` represents spans consisting of some number\nof *months* and *days*. This type is primarily intended to simplify logic\ninvolving :eql:type:`cal::local_date` values.\n\n.. code-block:: edgeql-repl\n\n db> select '5 days';\n {'P5D'}\n db> select '2022-06-25' + '5 days';\n {'2022-06-30'}\n db> select '2022-06-30' - '2022-06-25';\n {'P5D'}\n\nEdgeQL supports a set of functions and operators on duration types.\n\n.. list-table::\n\n * - Comparison operators\n - :eql:op:`= ` :eql:op:`\\!= ` :eql:op:`?= `\n :eql:op:`?!= ` :eql:op:`\\< ` :eql:op:`\\> `\n :eql:op:`\\<= ` :eql:op:`\\>= `\n * - Arithmetic\n - :eql:op:`dt + dt ` :eql:op:`dt - dt `\n * - Duration string parsing\n - :eql:func:`to_duration` :eql:func:`cal::to_relative_duration`\n :eql:func:`cal::to_date_duration`\n * - Component extraction\n - :eql:func:`duration_get`\n * - Conversion\n - :eql:func:`duration_truncate` :eql:func:`cal::duration_normalize_hours`\n :eql:func:`cal::duration_normalize_days`\n\n\n.. _ref_eql_ranges:\n\nRanges\n------\n\n.. api-index:: range, inc_lower, inc_upper, empty\n\nRanges represent a range of orderable scalar values. A range comprises a lower\nbound, upper bound, and two boolean flags indicating whether each bound is\ninclusive.\n\nCreate a range literal with the ``range`` constructor function.\n\n.. code-block:: edgeql-repl\n\n db> select range(1, 10);\n {range(1, 10, inc_lower := true, inc_upper := false)}\n db> select range(2.2, 3.3);\n {range(2.2, 3.3, inc_lower := true, inc_upper := false)}\n\nRanges can be *empty*, when the upper and lower bounds are equal.\n\n.. code-block:: edgeql-repl\n\n db> select range(1, 1);\n {range({}, empty := true)}\n\nRanges can be *unbounded*. An empty set is used to indicate the\nlack of a particular upper or lower bound.\n\n.. code-block:: edgeql-repl\n\n db> select range(4, {});\n {range(4, {})}\n db> select range({}, 4);\n {range({}, 4)}\n db> select range({}, {});\n {range({}, {})}\n\nTo compute the set of concrete values defined by a range literal, use\n``range_unpack``. An empty range will unpack to the empty set. Unbounded\nranges cannot be unpacked.\n\n.. code-block:: edgeql-repl\n\n db> select range_unpack(range(0, 10));\n {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}\n db> select range_unpack(range(1, 1));\n {}\n db> select range_unpack(range(0, {}));\n gel error: InvalidValueError: cannot unpack an unbounded range\n\n.. _ref_eql_literal_bytes:\n\nBytes\n-----\n\n.. index:: binary, raw byte strings\n.. api-index:: b'', b\"\", rb'', br'', rb\"\", br\"\"\n\nThe ``bytes`` type represents raw binary data.\n\n.. code-block:: edgeql-repl\n\n db> select b'bina\\\\x01ry';\n {b'bina\\\\x01ry'}\n\nThere is a special syntax for declaring \"raw byte strings\". Raw byte strings\ntreat the backslash ``\\`` as a literal character instead of an escape\ncharacter.\n\n.. code-block:: edgeql-repl\n\n db> select rb'hello\\nthere';\n {b'hello\\\\nthere'}\n db> select br'\\';\n {b'\\\\'}\n\n\n.. _ref_eql_literal_array:\n\nArrays\n------\n\n.. index:: collection, lists\n\nAn array is an *ordered* collection of values of the *same type*. For example:\n\n.. code-block:: edgeql-repl\n\n db> select [1, 2, 3];\n {[1, 2, 3]}\n db> select ['hello', 'world'];\n {['hello', 'world']}\n db> select [(1, 2), (100, 200)];\n {[(1, 2), (100, 200)]}\n\nEdgeQL provides a set of functions and operators on arrays.\n\n.. list-table::\n\n * - Indexing and slicing\n - :eql:op:`array[i] ` :eql:op:`array[from:to] `\n :eql:func:`array_get`\n * - Concatenation\n - :eql:op:`array ++ array `\n * - Comparison operators\n - :eql:op:`= ` :eql:op:`\\!= ` :eql:op:`?= `\n :eql:op:`?!= ` :eql:op:`\\< ` :eql:op:`\\> `\n :eql:op:`\\<= ` :eql:op:`\\>= `\n * - Utilities\n - :eql:func:`len` :eql:func:`array_join`\n * - Search\n - :eql:func:`contains` :eql:func:`find`\n * - Conversion to/from sets\n - :eql:func:`array_agg` :eql:func:`array_unpack`\n\nSee :ref:`Standard Library > Array ` for a complete\nreference on array data types.\n\n\n.. _ref_eql_literal_tuple:\n\nTuples\n------\n\nA tuple is *fixed-length*, *ordered* collection of values, each of which may\nhave a *different type*. The elements of a tuple can be of any type, including\nscalars, arrays, other tuples, and object types.\n\n.. code-block:: edgeql-repl\n\n db> select ('Apple', 7, true);\n {('Apple', 7, true)}\n\nOptionally, you can assign a key to each element of a tuple. These are known\nas *named tuples*. You must assign keys to all or none of the elements; you\ncan't mix-and-match.\n\n.. code-block:: edgeql-repl\n\n db> select (fruit := 'Apple', quantity := 3.14, fresh := true);\n {(fruit := 'Apple', quantity := 3.14, fresh := true)}\n\nIndexing tuples\n^^^^^^^^^^^^^^^\n\nTuple elements can be accessed with dot notation. Under the hood, there's no\ndifference between named and unnamed tuples. Named tuples support key-based\nand numerical indexing.\n\n.. code-block:: edgeql-repl\n\n db> select (1, 3.14, 'red').0;\n {1}\n db> select (1, 3.14, 'red').2;\n {'red'}\n db> select (name := 'george', age := 12).name;\n {('george')}\n db> select (name := 'george', age := 12).0;\n {('george')}\n\n.. important::\n\n When you query an *unnamed* tuple using one of EdgeQL's :ref:`client\n libraries `, its value is converted to a list/array. When\n you fetch a *named tuple*, it is converted to an object/dictionary/hashmap.\n\nFor a full reference on tuples, see :ref:`Standard Library > Tuple\n`.\n\n.. _ref_eql_literal_json:\n\nJSON\n----\n\nThe :eql:type:`json` scalar type is a stringified representation of structured\ndata. JSON literals are declared by explicitly casting other values or passing\na properly formatted JSON string into :eql:func:`to_json`. Any type can be\nconverted into JSON except :eql:type:`bytes`.\n\n.. code-block:: edgeql-repl\n\n db> select 5;\n {'5'}\n db> select \"a string\";\n {'\"a string\"'}\n db> select [\"this\", \"is\", \"an\", \"array\"];\n {'[\"this\", \"is\", \"an\", \"array\"]'}\n db> select (\"unnamed tuple\", 2);\n {'[\"unnamed tuple\", 2]'}\n db> select (name := \"named tuple\", count := 2);\n {'{\n \"name\": \"named tuple\",\n \"count\": 2\n }'}\n db> select to_json('{\"a\": 2, \"b\": 5}');\n {'{\"a\": 2, \"b\": 5}'}\n\nJSON values support indexing operators. The resulting value is also of type\n``json``.\n\n.. code-block:: edgeql-repl\n\n db> select to_json('{\"a\": 2, \"b\": 5}')['a'];\n {2}\n db> select to_json('[\"a\", \"b\", \"c\"]')[2];\n {'\"c\"'}\n\n\nEdgeQL supports a set of functions and operators on ``json`` values. Refer to\nthe :ref:`Standard Library > JSON ` or click an item below for\ndetailed documentation.\n\n.. list-table::\n\n * - Indexing\n - :eql:op:`json[i] ` :eql:op:`json[from:to] `\n :eql:op:`json[name] ` :eql:func:`json_get`\n * - Merging\n - :eql:op:`json ++ json `\n * - Comparison operators\n - :eql:op:`= ` :eql:op:`\\!= ` :eql:op:`?= `\n :eql:op:`?!= ` :eql:op:`\\< ` :eql:op:`\\> `\n :eql:op:`\\<= ` :eql:op:`\\>= `\n * - Conversion to/from strings\n - :eql:func:`to_json` :eql:func:`to_str`\n * - Conversion to/from sets\n - :eql:func:`json_array_unpack` :eql:func:`json_object_unpack`\n * - Introspection\n - :eql:func:`json_typeof`\n" }, { "path": "docs/reference/edgeql/parameters.rst", "content": ".. _ref_eql_params:\n\nParameters\n==========\n\n.. index:: query params, query arguments, query args, input\n.. api-index:: $, <Ā§typeĀ§>$\n\n:edb-alt-title: Query Parameters\n\nEdgeQL queries can reference parameters with ``$`` notation. The value of these\nparameters are supplied externally.\n\n.. code-block:: edgeql\n\n select $var;\n select $a + $b;\n select BlogPost filter .id = $blog_id;\n\nNote that we provided an explicit type cast before the parameter. This is\nrequired, as it enables Gel to enforce the provided types at runtime.\n\nParameters can be named or unnamed tuples.\n\n.. code-block:: edgeql\n\n select >$var;\n select >$var;\n select >$var;\n select >$var;\n\nUsage with clients\n------------------\n\nREPL\n^^^^\n\nWhen you include a parameter reference in a Gel REPL, you'll be prompted\ninteractively to provide a value or values.\n\n.. code-block:: edgeql-repl\n\n db> select 'I ā¤ļø ' ++ $var ++ '!';\n Parameter $var: Gel\n {'I ā¤ļø Gel!'}\n\n\nPython\n^^^^^^\n\n.. code-block:: python\n\n await client.query(\n \"select 'I ā¤ļø ' ++ $var ++ '!';\",\n var=\"lamp\")\n\n await client.query(\n \"select $date;\",\n date=datetime.today())\n\nJavaScript\n^^^^^^^^^^\n\n.. code-block:: javascript\n\n await client.query(\"select 'I ā¤ļø ' ++ $name ++ '!';\", {\n name: \"rock and roll\"\n });\n\n await client.query(\"select $date;\", {\n date: new Date()\n });\n\nGo\n^^\n\n.. code-block:: go\n\n var result string\n err = db.QuerySingle(ctx,\n `select 'I ā¤ļø ' ++ $var ++ '!';\"`,\n &result, \"Golang\")\n\n var date time.Time\n err = db.QuerySingle(ctx,\n `select $date;`,\n &date, time.Now())\n\n\nRefer to the Datatypes page of your preferred :ref:`client library\n` to learn more about mapping between Gel types and\nlanguage-native types.\n\n\n.. _ref_eql_params_types:\n\nParameter types and JSON\n------------------------\n\nIn Gel, parameters can also be tuples. If you need to pass complex structures\nas parameters, use Gel's built-in :ref:`JSON ` functionality.\n\n.. code-block:: edgeql-repl\n\n db> with data := $data\n ... insert Movie {\n ... title := data['title'],\n ... release_year := data['release_year'],\n ... };\n Parameter $data: {\"title\": \"The Marvels\", \"release_year\": 2023}\n {default::Movie {id: 8d286cfe-3c0a-11ec-aa68-3f3076ebd97f}}\n\nArrays can be \"unpacked\" into sets and assigned to ``multi`` links or\nproperties.\n\n.. code-block:: edgeql\n\n with friends := (\n select User filter .id in array_unpack(>$friend_ids)\n )\n insert User {\n name := $name,\n friends := friends,\n };\n\n\n.. _ref_eql_params_optional:\n\nOptional parameters\n-------------------\n\n.. api-index:: $\n\nBy default, query parameters are ``required``; the query will fail if the\nparameter value is an empty set. You can use an ``optional`` modifier inside\nthe type cast if the parameter is optional.\n\n.. code-block:: edgeql-repl\n\n db> select $name;\n Parameter $name (Ctrl+D for empty set `{}`):\n {}\n\n.. note::\n\n The ```` type cast is also valid (though redundant) syntax.\n\n .. code-block:: edgeql\n\n select $name;\n\n\nDefault parameter values\n------------------------\n\n.. api-index:: ??\n\nWhen using optional parameters, you may want to provide a default value to use\nin case the parameter is not passed. You can do this by using the\n:eql:op:`?? (coalesce) ` operator.\n\n.. code-block:: edgeql-repl\n\n db> select 'Hello ' ++ $name ?? 'there';\n Parameter $name (Ctrl+D for empty set `{}`): Gel\n {'Hello Gel'}\n db> select 'Hello ' ++ $name ?? 'there';\n Parameter $name (Ctrl+D for empty set `{}`):\n {'Hello there'}\n\n\nWhat can be parameterized?\n--------------------------\n\nAny data manipulation language (DML) statement can be\nparameterized: ``select``, ``insert``, ``update``, and ``delete``. Since\nparameters can only be scalars, arrays of scalars, and\ntuples of scalars, only parts of the query that would be one of those types can\nbe parameterized. This excludes parts of the query like the type being queried\nand the property to order by.\n\n.. note::\n\n You can parameterize ``order by`` for a limited number of options by using\n :eql:op:`if..else`:\n\n .. code-block:: edgeql\n\n select Movie {*}\n order by\n (.title if $order_by = 'title'\n else {})\n then\n (.release_year if $order_by = 'release_year'\n else {});\n\n If a user running this query enters ``title`` as the parameter value,\n ``Movie`` objects will be sorted by their ``title`` property. If they enter\n ``release_year``, they will be sorted by the ``release_year`` property.\n\n Since the ``if`` and ``else`` result clauses need to be of compatible\n types, your ``else`` expressions should be an empty set of the same type as\n the property.\n\nSchema definition language (SDL) and :ref:`configure\n` statements **cannot** be parameterized. Data\ndefinition language (DDL) has limited support for parameters, but it's not a\nrecommended pattern. Some of the limitations might be lifted in future\nversions.\n\n" }, { "path": "docs/reference/edgeql/path_resolution.rst", "content": ".. _ref_eql_path_resolution:\n\n============\nPath scoping\n============\n\n.. index:: using future simple_scoping, using future warn_old_scoping\n\nBeginning with Gel 6.0, we are phasing out our historical (and\nsomewhat notorious)\n:ref:`\"path scoping\" algorithm `\nin favor of a much simpler algorithm that nevertheless behaves\nidentically on *most* idiomatic EdgeQL queries.\n\nGel 6.0 will contain features to support migration to and testing\nof the new semantics. We expect the migration to be relatively\npainless for most users.\n\nDiscussion of rationale for this change is available in\n`the RFC `_.\n\n\nNew path scoping\n----------------\n\n.. versionadded:: 6.0\n\nWhen applying a shape to a path (or to a path that has shapes applied\nto it already), the path will be be bound inside computed\npointers in that shape:\n\n.. code-block:: edgeql-repl\n\n db> select User {\n ... name := User.first_name ++ ' ' ++ User.last_name\n ... }\n {User {name: 'Peter Parker'}, User {name: 'Tony Stark'}}\n\n\nWhen doing ``SELECT``, ``UPDATE``, or ``DELETE``, if the subject is a\npath, optionally with shapes applied to it, the path will be\nbound in ``FILTER`` and ``ORDER BY`` clauses:\n\n.. code-block:: edgeql-repl\n\n db> select User {\n ... name := User.first_name ++ ' ' ++ User.last_name\n ... }\n ... filter User.first_name = 'Peter'\n {User {name: 'Peter Parker'}}\n\n\nHowever, when a path is used multiple times in \"sibling\" contexts,\na cross-product will be computed:\n\n.. code-block:: edgeql-repl\n\n db> select User.first_name ++ ' ' ++ User.last_name;\n {'Peter Parker', 'Peter Stark', 'Tony Parker', 'Tony Stark'}\n\n\nIf you want to produce one value per ``User``, you can rewrite the query\nwith a ``FOR`` to make the intention explicit:\n\n.. code-block:: edgeql-repl\n\n db> for u in User\n ... select u.first_name ++ ' ' ++ u.last_name;\n {'Peter Parker', 'Tony Stark'}\n\nThe most idiomatic way to fetch such data in EdgeQL, however,\nremains:\n\n.. code-block:: edgeql-repl\n\n db> select User { name := .first_name ++ ' ' ++ .last_name }\n {User {name: 'Peter Parker'}, User {name: 'Tony Stark'}}\n\n(And, of course, you probably `shouldn't have first_name and last_name\nproperties anyway\n`_)\n\n\nPath scoping configuration\n--------------------------\n\n.. versionadded:: 6.0\n\nGel 6.0 introduces a new\n:ref:`future feature `\nnamed ``simple_scoping`` alongside a\nconfiguration setting also named ``simple_scoping``. The future\nfeature presence will determine which behavior is used inside\nexpressions within the schema, as well as serve as the default value\nif the configuration value is not set. The configuration setting will\nallow overriding the presence or absence of the feature.\n\nFor concreteness, here are all of the posible combinations of whether\n``using future simple_scoping`` is set and the value of the\nconfiguration value ``simple_scoping``:\n\n.. list-table::\n :widths: 25 25 25 25\n :header-rows: 1\n\n * - Future exists?\n - Config value\n - Query is simply scoped\n - Schema is simply scoped\n * - No\n - ``{}``\n - No\n - No\n * - No\n - ``true``\n - Yes\n - No\n * - No\n - ``false``\n - No\n - No\n * - Yes\n - ``{}``\n - Yes\n - Yes\n * - Yes\n - ``true``\n - Yes\n - Yes\n * - Yes\n - ``false``\n - No\n - Yes\n\n.. _ref_warn_old_scoping:\n\nWarning on old scoping\n----------------------\n\n.. versionadded:: 6.0\n\nTo make the migration process safer, we have also introduced a\n``warn_old_scoping`` :ref:`future feature ` and\nconfig setting.\n\nWhen active, the server will emit a warning to the client when a query\nis detected to depend on the old scoping behavior. The behavior of\nwarnings can be configured in client bindings, but by default they are\nlogged.\n\nThe check is known to sometimes produce false positives, on queries\nthat will not actually have changed behavior, but is intended to not\nhave false negatives.\n\nRecommended upgrade plan\n------------------------\n\n.. versionadded:: 6.0\n\nThe safest approach is to first get your entire schema and application\nworking with ``warn_old_scoping`` without producing any warnings. Once\nthat is done, it should be safe to switch to ``simple_scoping``\nwithout changes in behavior.\n\nIf you are very confident in your test coverage, though, you can try\nskipping dealing with ``warn_old_scoping`` and go straight to\n``simple_scoping``.\n\nThere are many different potential migration strategies. One that\nshould work well:\n\n1. Run ``CONFIGURE CURRENT DATABASE SET warn_old_scoping := true``\n2. Try running all of your queries against the database.\n3. Fix any that produce warnings.\n4. Adjust your schema until setting ``using future warn_old_scoping`` works\n without producing warnings.\n\nIf you wish to proceed incrementally with steps 2 and 3, you can\nconfigure ``warn_old_scoping`` in your clients, having it enabled for\nqueries that you have verified work with it and disabled for queries\nthat have not yet been verified or updated.\n\n\n.. _ref_eql_old_path_resolution:\n\nLegacy path scoping\n-------------------\n\nThis section describes the path scoping algorithm used exclusively\nuntil |EdgeDB| 5.0 and by default in |Gel| 6.0.\nIt will be removed in Gel 7.0.\n\nElement-wise operations with multiple arguments in Gel are generally applied\nto the :ref:`cartesian product ` of all\nthe input sets.\n\n.. code-block:: edgeql-repl\n\n db> select {'aaa', 'bbb'} ++ {'ccc', 'ddd'};\n {'aaaccc', 'aaaddd', 'bbbccc', 'bbbddd'}\n\nHowever, in cases where multiple element-wise arguments share a common path\n(``User.`` in this example), Gel factors out the common path rather than\nusing cartesian multiplication.\n\n.. code-block:: edgeql-repl\n\n db> select User.first_name ++ ' ' ++ User.last_name;\n {'Mina Murray', 'Jonathan Harker', 'Lucy Westenra', 'John Seward'}\n\nWe assume this is what you want, but if your goal is to get the cartesian\nproduct, you can accomplish it one of three ways. You could use\n:eql:op:`detached`.\n\n.. code-block:: edgeql-repl\n\n gel> select User.first_name ++ ' ' ++ detached User.last_name;\n {\n 'Mina Murray',\n 'Mina Harker',\n 'Mina Westenra',\n 'Mina Seward',\n 'Jonathan Murray',\n 'Jonathan Harker',\n 'Jonathan Westenra',\n 'Jonathan Seward',\n 'Lucy Murray',\n 'Lucy Harker',\n 'Lucy Westenra',\n 'Lucy Seward',\n 'John Murray',\n 'John Harker',\n 'John Westenra',\n 'John Seward',\n }\n\nYou could use :ref:`with ` to attach a different symbol to\nyour set of ``User`` objects.\n\n.. code-block:: edgeql-repl\n\n gel> with U := User\n .... select U.first_name ++ ' ' ++ User.last_name;\n {\n 'Mina Murray',\n 'Mina Harker',\n 'Mina Westenra',\n 'Mina Seward',\n 'Jonathan Murray',\n 'Jonathan Harker',\n 'Jonathan Westenra',\n 'Jonathan Seward',\n 'Lucy Murray',\n 'Lucy Harker',\n 'Lucy Westenra',\n 'Lucy Seward',\n 'John Murray',\n 'John Harker',\n 'John Westenra',\n 'John Seward',\n }\n\nOr you could leverage the effect scopes have on path resolution. More on that\n:ref:`in the Scopes section `.\n\nThe reason ``with`` works here even though the alias ``U`` refers to the exact\nsame set is that we only assume you want the path factored in this way when you\nuse the same *symbol* to refer to a set. This means operations with\n``User.first_name`` and ``User.last_name`` *do* get the common path factored\nwhile ``U.first_name`` and ``User.last_name`` *do not* and are resolved with\ncartesian multiplication.\n\nThat may leave you still wondering why ``U`` and ``User`` did not get a common\npath factored. ``U`` is just an alias of ``select User`` and ``User`` is the\nsame symbol that we use in our name query. That's true, but |Gel| doesn't\nfactor in this case because of the queries' scopes.\n\n.. _ref_eql_path_resolution_scopes:\n\nScopes\n------\n\nScopes change the way path resolution works. Two sibling select queries ā€” that\nis, queries at the same level ā€” do not have their paths factored even when they\nuse a common symbol.\n\n.. code-block:: edgeql-repl\n\n gel> select ((select User.first_name), (select User.last_name));\n {\n ('Mina', 'Murray'),\n ('Mina', 'Harker'),\n ('Mina', 'Westenra'),\n ('Mina', 'Seward'),\n ('Jonathan', 'Murray'),\n ('Jonathan', 'Harker'),\n ('Jonathan', 'Westenra'),\n ('Jonathan', 'Seward'),\n ('Lucy', 'Murray'),\n ('Lucy', 'Harker'),\n ('Lucy', 'Westenra'),\n ('Lucy', 'Seward'),\n ('John', 'Murray'),\n ('John', 'Harker'),\n ('John', 'Westenra'),\n ('John', 'Seward'),\n }\n\nCommon symbols in nested scopes *are* factored when they use the same symbol.\nIn this example, the nested queries both use the same ``User`` symbol as the\ntop-level query. As a result, the ``User`` in those queries refers to a single\nobject because it has been factored.\n\n.. code-block:: edgeql-repl\n\n gel> select User {\n .... name:= (select User.first_name) ++ ' ' ++ (select User.last_name)\n .... };\n {\n default::User {name: 'Mina Murray'},\n default::User {name: 'Jonathan Harker'},\n default::User {name: 'Lucy Westenra'},\n default::User {name: 'John Seward'},\n }\n\nIf you have two common scopes and only *one* of them is in a nested scope, the\npaths are still factored.\n\n.. code-block:: edgeql-repl\n\n gel> select (Person.name, count(Person.friends));\n {('Fran', 3), ('Bam', 2), ('Emma', 3), ('Geoff', 1), ('Tyra', 1)}\n\nIn this example, ``count``, like all aggregate function, creates a nested\nscope, but this doesn't prevent the paths from being factored as you can see\nfrom the results. If the paths were *not* factored, the friend count would be\nthe same for all the result tuples and it would reflect the total number of\n``Person`` objects that are in *all* ``friends`` links rather than the number\nof ``Person`` objects that are in the named ``Person`` object's ``friends``\nlink.\n\nIf you have two aggregate functions creating *sibling* nested scopes, the paths\nare *not* factored.\n\n.. code-block:: edgeql-repl\n\n gel> select (array_agg(distinct Person.name), count(Person.friends));\n {(['Fran', 'Bam', 'Emma', 'Geoff'], 3)}\n\nThis query selects a tuple containing two nested scopes. Here, |Gel| assumes\nyou want an array of all unique names and a count of the total number of people\nwho are anyone's friend.\n\nClauses & Nesting\n^^^^^^^^^^^^^^^^^\n\nMost clauses are nested and are subjected to the same rules described above:\ncommon symbols are factored and assumed to refer to the same object as the\nouter query. This is because clauses like :ref:`filter\n` and :ref:`order by ` need to\nbe applied to each value in the result.\n\nThe :ref:`offset ` and\n:ref:`limit ` clauses are not nested in the scope\nbecause they need to be applied globally to the entire result set of your\nquery.\n\n.. _rfc: https://github.com/geldata/rfcs/blob/master/text/1027-no-factoring.rst\n" }, { "path": "docs/reference/edgeql/paths.rst", "content": ".. _ref_eql_paths:\n\n=====\nPaths\n=====\n\n.. index:: links, relations\n\nA *path expression* (or simply a *path*) represents a set of values that are\nreachable by traversing a given sequence of links or properties from some\nsource set of objects.\n\nConsider the following schema:\n\n.. code-block:: sdl\n\n type User {\n required email: str;\n multi friends: User;\n }\n\n type BlogPost {\n required title: str;\n required author: User;\n }\n\n type Comment {\n required text: str;\n required author: User;\n }\n\nA few simple inserts will allow some experimentation with paths.\n\nStart with a first user:\n\n.. code-block:: edgeql-repl\n\n db> insert User {\n ... email := \"user1@me.com\",\n ... };\n\nAlong comes another user who adds the first user as a friend:\n\n.. code-block:: edgeql-repl\n\n db> insert User {\n ... email := \"user2@me.com\",\n ... friends := (select detached User filter .email = \"user1@me.com\")\n ... };\n\nThe first user reciprocates, adding the new user as a friend:\n\n.. code-block:: edgeql-repl\n\n db> update User filter .email = \"user1@me.com\"\n ... set {\n ... friends += (select detached User filter .email = \"user2@me.com\")\n ... };\n\nThe second user writes a blog post about how nice Gel is:\n\n.. code-block:: edgeql-repl\n\n db> insert BlogPost {\n ... title := \"Gel is awesome\",\n ... author := assert_single((select User filter .email = \"user2@me.com\"))\n ... };\n\nAnd the first user follows it up with a comment below the post:\n\n.. code-block:: edgeql-repl\n\n db> insert Comment {\n ... text := \"Nice post, user2!\",\n ... author := assert_single((select User filter .email = \"user1@me.com\"))\n ... };\n\nThe simplest path is simply ``User``. This is a :ref:`set reference\n` that refers to all ``User`` objects in the database.\n\n.. code-block:: edgeql\n\n select User;\n\nPaths can traverse links. The path below refers to *all Users who are the\nfriend of another User*.\n\n.. code-block:: edgeql\n\n select User.friends;\n\nPaths can traverse to an arbitrary depth in a series of nested links.\nBoth ``select`` queries below end up showing the author of the ``BlogPost``.\nThe second query returns the friends of the friends of the author of the\n``BlogPost``, which in this case is just the author.\n\n.. code-block:: edgeql\n\n select BlogPost.author; # The author\n select BlogPost.author.friends.friends; # The author again\n\nPaths can terminate with a property reference.\n\n.. code-block:: edgeql\n\n select BlogPost.title; # all blog post titles\n select BlogPost.author.email; # all author emails\n select User.friends.email; # all friends' emails\n\n.. _ref_eql_paths_backlinks:\n\nBacklinks\n---------\n\n.. api-index:: .<\n\nAll examples thus far have traversed links in the *forward direction*, however\nit's also possible to traverse links *backwards* with ``.<`` notation. These\nare called **backlinks**.\n\nStarting from each user, the path below traverses all *incoming* links labeled\n``author`` and returns the union of their sources.\n\n.. code-block:: edgeql\n\n select User.` operator:\n``[is Foo]``:\n\n.. code-block:: edgeql\n\n # BlogPost objects that link to the user via a link named author\n select User.`\nwith ``@`` notation. To demonstrate this, let's add a property to the ``User.\nfriends`` link:\n\n.. code-block:: sdl-diff\n\n type User {\n required email: str;\n - multi friends: User;\n + multi friends: User {\n + since: cal::local_date;\n + }\n }\n\nThe following represents a set of all dates on which friendships were formed.\n\n.. code-block:: edgeql\n\n select User.friends@since;\n\nPath roots\n----------\n\nFor simplicity, all examples above use set references like ``User`` as the root\nof the path; however, the root can be *any expression* returning object types.\nBelow, the root of the path is a *subquery*.\n\n.. code-block:: edgeql-repl\n\n db> with gel_lovers := (\n ... select BlogPost filter .title ilike \"Gel is awesome\"\n ... )\n ... select gel_lovers.author;\n\nThis expression returns a set of all ``Users`` who have written a blog post\ntitled \"Gel is awesome\".\n\nFor a full syntax definition, see the :ref:`Reference > Paths\n`.\n" }, { "path": "docs/reference/edgeql/select.rst", "content": ".. _ref_eql_select:\n\nSelect\n======\n\n.. api-index:: select\n\nThe ``select`` command retrieves or computes a set of values. We've already\nseen simple queries that select primitive values.\n\n.. code-block:: edgeql-repl\n\n db> select 'hello world';\n {'hello world'}\n db> select [1, 2, 3];\n {[1, 2, 3]}\n db> select {1, 2, 3};\n {1, 2, 3}\n\n\nWith the help of a ``with`` block, we can add filters, ordering, and\npagination clauses.\n\n.. code-block:: edgeql-repl\n\n db> with x := {1, 2, 3, 4, 5}\n ... select x\n ... filter x >= 3;\n {3, 4, 5}\n db> with x := {1, 2, 3, 4, 5}\n ... select x\n ... order by x desc;\n {5, 4, 3, 2, 1}\n db> with x := {1, 2, 3, 4, 5}\n ... select x\n ... offset 1 limit 3;\n {2, 3, 4}\n\nThese queries can also be rewritten to use inline aliases, like so:\n\n.. code-block:: edgeql-repl\n\n db> select x := {1, 2, 3, 4, 5}\n ... filter x >= 3;\n\n\n.. _ref_eql_select_objects:\n\nSelecting objects\n-----------------\n\nHowever most queries are selecting *objects* that live in the database. For\ndemonstration purposes, the queries below assume the following schema:\n\n.. code-block:: sdl\n\n module default {\n abstract type Person {\n required name: str { constraint exclusive };\n }\n\n type Hero extending Person {\n secret_identity: str;\n multi villains := . insert Hero {\n ... name := \"Spider-Man\",\n ... secret_identity := \"Peter Parker\"\n ... };\n {default::Hero {id: 6be1c9c6...}}\n\n db> insert Hero {\n ... name := \"Iron Man\",\n ... secret_identity := \"Tony Stark\"\n ... };\n {default::Hero {id: 6bf7115a... }}\n\n db> for n in { \"Sandman\", \"Electro\", \"Green Goblin\", \"Doc Ock\" }\n ... union (\n ... insert Villain {\n ... name := n,\n ... nemesis := (select Hero filter .name = \"Spider-Man\")\n ... });\n {\n default::Villain {id: 6c22bdf0...},\n default::Villain {id: 6c22c3d6...},\n default::Villain {id: 6c22c46c...},\n default::Villain {id: 6c22c502...},\n }\n\n db> insert Villain {\n ... name := \"Obadiah Stane\",\n ... nemesis := (select Hero filter .name = \"Iron Man\")\n ... };\n {default::Villain {id: 6c42c4ec...}}\n\n db> insert Movie {\n ... title := \"Spider-Man: No Way Home\",\n ... release_year := 2021,\n ... characters := (select Person filter .name in\n ... { \"Spider-Man\", \"Sandman\", \"Electro\", \"Green Goblin\", \"Doc Ock\" })\n ... };\n {default::Movie {id: 6c60c28a...}}\n\n db> insert Movie {\n ... title := \"Iron Man\",\n ... release_year := 2008,\n ... characters := (select Person filter .name in\n ... { \"Iron Man\", \"Obadiah Stane\" })\n ... };\n {default::Movie {id: 6d1f430e...}}\n\nLet's start by selecting all ``Villain`` objects in the database. In this\nexample, there are only five. Remember, ``Villain`` is a :ref:`reference\n` to the set of all Villain objects.\n\n.. code-block:: edgeql-repl\n\n db> select Villain;\n {\n default::Villain {id: 6c22bdf0...},\n default::Villain {id: 6c22c3d6...},\n default::Villain {id: 6c22c46c...},\n default::Villain {id: 6c22c502...},\n default::Villain {id: 6c42c4ec...},\n }\n\n.. note::\n\n For the sake of readability, the ``id`` values have been truncated.\n\nBy default, this only returns the ``id`` of each object. If serialized to JSON,\nthis result would look like this:\n\n.. code-block::\n\n [\n {\"id\": \"6c22bdf0-5c03-11ee-99ff-dfaea4d947ce\"},\n {\"id\": \"6c22c3d6-5c03-11ee-99ff-734255881e5d\"},\n {\"id\": \"6c22c46c-5c03-11ee-99ff-c79f24cf638b\"},\n {\"id\": \"6c22c502-5c03-11ee-99ff-cbacc3918129\"},\n {\"id\": \"6c42c4ec-5c03-11ee-99ff-872c9906a467\"}\n ]\n\n\n.. _ref_eql_shapes:\n\nShapes\n------\n\n.. api-index:: select, { }\n\nTo specify which properties to select, we attach a **shape** to ``Villain``. A\nshape can be attached to any object type expression in EdgeQL.\n\n.. code-block:: edgeql-repl\n\n db> select Villain { id, name };\n {\n default::Villain {id: 6c22bdf0..., name: 'Sandman'},\n default::Villain {id: 6c22c3d6..., name: 'Electro'},\n default::Villain {id: 6c22c46c..., name: 'Green Goblin'},\n default::Villain {id: 6c22c502..., name: 'Doc Ock'},\n default::Villain {id: 6c42c4ec..., name: 'Obadiah Stane'},\n }\n\nNested shapes\n^^^^^^^^^^^^^\n\nNested shapes can be used to fetch linked objects and their properties. Here we\nfetch all ``Villain`` objects and their nemeses.\n\n.. code-block:: edgeql-repl\n\n db> select Villain {\n ... name,\n ... nemesis: { name }\n ... };\n {\n default::Villain {\n name: 'Sandman',\n nemesis: default::Hero {name: 'Spider-Man'},\n },\n ...\n }\n\nIn the context of EdgeQL, computed links like ``Hero.villains`` are treated\nidentically to concrete/non-computed links like ``Villain.nemesis``.\n\n.. code-block:: edgeql-repl\n\n db> select Hero {\n ... name,\n ... villains: { name }\n ... };\n {\n default::Hero {\n name: 'Spider-Man',\n villains: {\n default::Villain {name: 'Sandman'},\n default::Villain {name: 'Electro'},\n default::Villain {name: 'Green Goblin'},\n default::Villain {name: 'Doc Ock'},\n },\n },\n ...\n }\n\n\n.. _ref_eql_select_splats:\n\nSplats\n^^^^^^\n\n.. index:: select *, select all\n.. api-index:: *, **, Ā§typeĀ§.*, Ā§typeĀ§.**, [is Ā§typeĀ§].*, [is Ā§typeĀ§].**\n\nSplats allow you to select all properties of a type using the asterisk (``*``)\nor all properties of the type and a single level of linked types with a double\nasterisk (``**``).\n\n.. edb:youtube-embed:: 9-I1qjIp3KI\n\nSplats will help you more easily select all properties when using the REPL.\nYou can select all of an object's properties using the single splat:\n\n.. code-block:: edgeql-repl\n\n db> select Movie {*};\n {\n default::Movie {\n id: 6c60c28a-5c03-11ee-99ff-dfa425012a05,\n release_year: 2021,\n title: 'Spider-Man: No Way Home',\n },\n default::Movie {\n id: 6d1f430e-5c03-11ee-99ff-e731e8da06d9,\n release_year: 2008,\n title: 'Iron Man'\n },\n }\n\nor you can select all of an object's properties and the properties of a single\nlevel of nested objects with the double splat:\n\n.. code-block:: edgeql-repl\n\n db> select Movie {**};\n {\n default::Movie {\n id: 6c60c28a-5c03-11ee-99ff-dfa425012a05,\n release_year: 2021,\n title: 'Spider-Man: No Way Home',\n characters: {\n default::Hero {\n id: 6be1c9c6-5c03-11ee-99ff-63b1127d75f2,\n name: 'Spider-Man'\n },\n default::Villain {\n id: 6c22bdf0-5c03-11ee-99ff-dfaea4d947ce,\n name: 'Sandman'\n },\n default::Villain {\n id: 6c22c3d6-5c03-11ee-99ff-734255881e5d,\n name: 'Electro'\n },\n default::Villain {\n id: 6c22c46c-5c03-11ee-99ff-c79f24cf638b,,\n name: 'Green Goblin'\n },\n default::Villain {\n id: 6c22c502-5c03-11ee-99ff-cbacc3918129,\n name: 'Doc Ock'\n },\n },\n },\n default::Movie {\n id: 6d1f430e-5c03-11ee-99ff-e731e8da06d9,\n release_year: 2008,\n title: 'Iron Man',\n characters: {\n default::Hero {\n id: 6bf7115a-5c03-11ee-99ff-c79c07f0e2db,\n name: 'Iron Man'\n },\n default::Villain {\n id: 6c42c4ec-5c03-11ee-99ff-872c9906a467,\n name: 'Obadiah Stane'\n },\n },\n },\n }\n\n.. note::\n\n Splats are not yet supported in function bodies.\n\nThe splat expands all properties defined on the type as well as inherited\nproperties:\n\n.. code-block:: edgeql-repl\n\n db> select Hero {*};\n {\n default::Hero {\n id: 6be1c9c6-5c03-11ee-99ff-63b1127d75f2,\n name: 'Spider-Man',\n secret_identity: 'Peter Parker'\n },\n default::Hero {\n id: 6bf7115a-5c03-11ee-99ff-c79c07f0e2db,\n name: 'Iron Man',\n secret_identity: 'Tony Stark'\n },\n }\n\nThe splat here expands the heroes' names even though the ``name`` property is\nnot defined on the ``Hero`` type but on the ``Person`` type it extends. If we\nwant to select heroes but get only properties defined on the ``Person`` type,\nwe can do this instead:\n\n.. code-block:: edgeql-repl\n\n db> select Hero {Person.*};\n {\n default::Hero {\n id: 6be1c9c6-5c03-11ee-99ff-63b1127d75f2,\n name: 'Spider-Man'\n },\n default::Hero {\n id: 6bf7115a-5c03-11ee-99ff-c79c07f0e2db,\n name: 'Iron Man'\n },\n }\n\nIf there are links on our ``Person`` type, we can use ``Person.**`` in a\nsimilar fashion to get all properties and one level of linked object\nproperties, but only for links and properties that are defined on the\n``Person`` type.\n\nYou can use the splat to expand properties using a :ref:`type intersection\n`. Maybe we want to select all ``Person`` objects\nwith their names but also get any properties defined on the ``Hero`` for those\n``Person`` objects which are also ``Hero`` objects:\n\n.. code-block:: edgeql-repl\n\n db> select Person {\n ... name,\n ... [is Hero].*\n ... };\n {\n default::Hero {\n name: 'Spider-Man',\n id: 6be1c9c6-5c03-11ee-99ff-63b1127d75f2,\n secret_identity: 'Peter Parker'\n },\n default::Hero {\n name: 'Iron Man'\n id: 6bf7115a-5c03-11ee-99ff-c79c07f0e2db,\n secret_identity: 'Tony Stark'\n },\n default::Villain {\n name: 'Sandman',\n id: 6c22bdf0-5c03-11ee-99ff-dfaea4d947ce,\n secret_identity: {}\n },\n default::Villain {\n name: 'Electro',\n id: 6c22c3d6-5c03-11ee-99ff-734255881e5d,\n secret_identity: {}\n },\n default::Villain {\n name: 'Green Goblin',\n id: 6c22c46c-5c03-11ee-99ff-c79f24cf638b,\n secret_identity: {}\n },\n default::Villain {\n name: 'Doc Ock',\n id: 6c22c502-5c03-11ee-99ff-cbacc3918129,\n secret_identity: {}\n },\n default::Villain {\n name: 'Obadiah Stane',\n id: 6c42c4ec-5c03-11ee-99ff-872c9906a467,\n secret_identity: {}\n },\n }\n\nThe double splat also works with type intersection expansion to expand both\nproperties and links on the specified type.\n\n.. code-block:: edgeql-repl\n\n db> select Person {\n ... name,\n ... [is Hero].**\n ... };\n {\n default::Villain {\n name: 'Sandman',\n id: 6c22bdf0-5c03-11ee-99ff-dfaea4d947ce,\n secret_identity: {},\n villains: {}\n },\n default::Villain {\n name: 'Electro',\n id: 6c22c3d6-5c03-11ee-99ff-734255881e5d,\n secret_identity: {},\n villains: {}\n },\n default::Villain {\n name: 'Green Goblin',\n id: 6c22c46c-5c03-11ee-99ff-c79f24cf638b,\n secret_identity: {},\n villains: {}\n },\n default::Villain {\n name: 'Doc Ock',\n id: 6c22c502-5c03-11ee-99ff-cbacc3918129,\n secret_identity: {},\n villains: {}\n },\n default::Villain {\n name: 'Obadiah Stane',\n id: 6c42c4ec-5c03-11ee-99ff-872c9906a467,\n secret_identity: {},\n villains: {}\n },\n default::Hero {\n name: 'Spider-Man',\n id: 6be1c9c6-5c03-11ee-99ff-63b1127d75f2,\n secret_identity: 'Peter Parker',\n villains: {\n default::Villain {\n name: 'Electro',\n id: 6c22c3d6-5c03-11ee-99ff-734255881e5d\n },\n default::Villain {\n name: 'Sandman',\n id: 6c22bdf0-5c03-11ee-99ff-dfaea4d947ce\n },\n default::Villain {\n name: 'Doc Ock',\n id: 6c22c502-5c03-11ee-99ff-cbacc3918129\n },\n default::Villain {\n name: 'Green Goblin',\n id: 6c22c46c-5c03-11ee-99ff-c79f24cf638b\n },\n },\n },\n }\n\nWith this query, we get ``name`` for each ``Person`` and all the properties and\none level of links on the ``Hero`` objects. We don't get ``Villain`` objects'\nnemeses because that link is not covered by our double splat which only\nexpands ``Hero`` links. If the ``Villain`` type had properties defined on it,\nwe wouldn't get those with this query either.\n\n\n.. _ref_eql_select_filter:\n\nFiltering\n---------\n\n.. index:: where\n.. api-index:: filter\n\nTo filter the set of selected objects, use a ``filter `` clause. The\n```` that follows the ``filter`` keyword can be *any boolean expression*.\n\nTo reference the ``name`` property of the ``Villain`` objects being selected,\nwe use ``Villain.name``.\n\n.. code-block:: edgeql-repl\n\n db> select Villain {id, name}\n ... filter Villain.name = \"Doc Ock\";\n {default::Villain {id: 6c22c502..., name: 'Doc Ock'}}\n\n\n.. note::\n\n This query contains two occurrences of ``Villain``. The first\n (outer) is passed as the argument to ``select`` and refers to the set of all\n ``Villain`` objects. However the *inner* occurrence is inside the *scope* of\n the ``select`` statement and refers to the *object being\n selected*.\n\nHowever, this looks a little clunky, so EdgeQL provides a shorthand: just drop\n``Villain`` entirely and simply use ``.name``. Since we are selecting a set of\nVillains, it's clear from context that ``.name`` must refer to a link/property\nof the ``Villain`` type. In other words, we are in the **scope** of the\n``Villain`` type.\n\n.. code-block:: edgeql-repl\n\n db> select Villain {name}\n ... filter .name = \"Doc Ock\";\n {default::Villain {name: 'Doc Ock'}}\n\n.. warning::\n\n When using comparison operators like ``=`` or ``!=``, or boolean operators\n ``and``, ``or``, and ``not``, keep in mind that these operators will\n produce an empty set if an operand is an empty set. Check out :ref:`our\n boolean cheatsheet ` for more info and help on how\n to mitigate this if you know your operands may be an empty set.\n\nFiltering by ID\n^^^^^^^^^^^^^^^\n\nTo filter by ``id``, remember to cast the desired ID to :ref:`uuid\n`:\n\n.. code-block:: edgeql-repl\n\n db> select Villain {id, name}\n ... filter .id = \"6c22c502-5c03-11ee-99ff-cbacc3918129\";\n {\n default::Villain {\n id: '6c22c502-5c03-11ee-99ff-cbacc3918129',\n name: 'Doc Ock'\n }\n }\n\nNested filters\n^^^^^^^^^^^^^^\n\nFilters can be added at every level of shape nesting. The query below applies a\nfilter to both the selected ``Hero`` objects and their linked ``villains``.\n\n.. code-block:: edgeql-repl\n\n db> select Hero {\n ... name,\n ... villains: {\n ... name\n ... } filter .name like \"%O%\"\n ... } filter .name ilike \"%man\";\n {\n default::Hero {\n name: 'Spider-Man',\n villains: {\n default::Villain {\n name: 'Doc Ock'\n }\n }\n },\n default::Hero {\n name: 'Iron Man',\n villains: {\n default::Villain {\n name: 'Obadiah Stane'\n }\n }\n },\n }\n\nNote that the *scope* changes inside nested shapes. When we use ``.name`` in\nthe outer ``filter``, it refers to the name of the hero. But when we use\n``.name`` in the nested ``villains`` shape, the scope has changed to\n``Villain``.\n\nFiltering on a known backlink\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nAnother handy use for backlinks is using them to filter and find items\nwhen doing a ``select`` (or an ``update`` or other operation, of course).\nThis can work as a nice shortcut when you have the ID of one object that\nlinks to a second object without a link back to the first.\n\nSpider-Man's villains always have a grudging respect for him, and their names\ncan be displayed to reflect that if we know the ID of a movie that they\nstarred in. Note the ability to :ref:`cast from a uuid `\nto an object type.\n\n.. code-block:: edgeql-repl\n\n db> select Villain filter .'6c60c28a-5c03-11ee-99ff-dfa425012a05' {\n ... name := .name ++ ', who got to see Spider-Man!'\n ... };\n {\n 'Obadiah Stane',\n 'Sandman, who got to see Spider-Man!',\n 'Electro, who got to see Spider-Man!',\n 'Green Goblin, who got to see Spider-Man!',\n 'Doc Ock, who got to see Spider-Man!',\n }\n\nIn other words, \"select every ``Villain`` object that the ``Movie`` object\nof this ID links to via a link called ``characters``\".\n\nA backlink is naturally not required, however. The same operation without\ntraversing a backlink would look like this:\n\n.. code-block:: edgeql-repl\n\n db> with movie :=\n ... '6c60c28a-5c03-11ee-99ff-dfa425012a05',\n ... select movie.characters[is Villain] {\n ... name := .name ++ ', who got to see Spider-Man!'\n ... };\n\n\n.. _ref_eql_select_order:\n\nFiltering, ordering, and limiting of links\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nClauses like ``filter``, ``order by``, and ``limit`` can be used on links.\nIf no properties of a link are selected, you can place the clauses directly\ninside the shape:\n\n.. code-block:: edgeql\n\n select User {\n likes order by .title desc limit 10\n };\n\nIf properties are selected, place the clauses after the link's shape:\n\n.. code-block:: edgeql\n\n select User {\n likes: {\n id,\n title\n } order by .title desc limit 10\n };\n\n\nOrdering\n--------\n\n.. index:: sorting\n.. api-index:: order by, asc, desc, then, empty first, empty last\n\nOrder the result of a query with an ``order by`` clause.\n\n.. code-block:: edgeql-repl\n\n db> select Villain { name }\n ... order by .name;\n {\n default::Villain {name: 'Doc Ock'},\n default::Villain {name: 'Electro'},\n default::Villain {name: 'Green Goblin'},\n default::Villain {name: 'Obadiah Stane'},\n default::Villain {name: 'Sandman'},\n }\n\nThe expression provided to ``order by`` may be *any* singleton\nexpression, primitive or otherwise.\n\n.. note::\n\n In Gel all values are orderable. Objects are compared using their ``id``;\n tuples and arrays are compared element-by-element from left to right. By\n extension, the generic comparison operators :eql:op:`= `,\n :eql:op:`\\< `, :eql:op:`\\> `, etc. can be used with any two\n expressions of the same type.\n\nYou can also order by multiple\nexpressions and specify the *direction* with an ``asc`` (default) or ``desc``\nmodifier.\n\n.. note::\n\n When ordering by multiple expressions, arrays, or tuples, the leftmost\n expression/element is compared. If these elements are the same, the next\n element is used to \"break the tie\", and so on. If all elements are the same,\n the order is not well defined.\n\n.. code-block:: edgeql-repl\n\n db> select Movie { title, release_year }\n ... order by\n ... .release_year desc then\n ... str_trim(.title) desc;\n {\n default::Movie {title: 'Spider-Man: No Way Home', release_year: 2021},\n ...\n default::Movie {title: 'Iron Man', release_year: 2008},\n }\n\nWhen ordering by multiple expressions, each expression is separated with the\n``then`` keyword. For a full reference on ordering, including how empty values\nare handled, see :ref:`Reference > Commands > Select\n`.\n\n\n.. _ref_eql_select_pagination:\n\nPagination\n----------\n\n.. api-index:: limit, offset\n\n|Gel| supports ``limit`` and ``offset`` clauses. These are\ntypically used in conjunction with ``order by`` to maintain a consistent\nordering across pagination queries.\n\n.. code-block:: edgeql-repl\n\n db> select Villain { name }\n ... order by .name\n ... offset 2\n ... limit 2;\n {\n default::Villain {name: 'Obadiah Stane'},\n default::Villain {name: 'Sandman'},\n }\n\nThe expressions passed to ``limit`` and ``offset`` can be any singleton\n``int64`` expression. This query fetches all Villains except the last (sorted\nby name).\n\n.. code-block:: edgeql-repl\n\n db> select Villain {name}\n ... order by .name\n ... limit count(Villain) - 1;\n {\n default::Villain {name: 'Doc Ock'},\n default::Villain {name: 'Electro'},\n default::Villain {name: 'Green Goblin'},\n default::Villain {name: 'Obadiah Stane'}, # no Sandman\n }\n\nYou may pass the empty set to ``limit`` or ``offset``. Passing the empty set is\neffectively the same as excluding ``limit`` or ``offset`` from your query\n(i.e., no limit or no offset). This is useful if you need to parameterize\n``limit`` and/or ``offset`` but may still need to execute your query without\nproviding one or the other.\n\n.. code-block:: edgeql-repl\n\n db> select Villain {name}\n ... order by .name\n ... offset $offset\n ... limit $limit;\n Parameter $offset (Ctrl+D for empty set `{}`):\n Parameter $limit (Ctrl+D for empty set `{}`):\n {\n default::Villain {name: 'Doc Ock'},\n default::Villain {name: 'Electro'},\n ...\n }\n\n.. note::\n\n If you parameterize ``limit`` and ``offset`` and want to reserve the option\n to pass the empty set, make sure those parameters are ``optional`` as shown\n in the example above.\n\n\n.. _ref_eql_select_computeds:\n\nComputed fields\n---------------\n\n.. api-index:: :=\n\nShapes can contain *computed fields*. These are EdgeQL expressions that are\ncomputed on the fly during the execution of the query. As with other clauses,\nwe can use :ref:`leading dot notation ` (e.g. ``.name``) to\nrefer to the properties and links of the object type currently *in scope*.\n\n\n.. code-block:: edgeql-repl\n\n db> select Villain {\n ... name,\n ... name_upper := str_upper(.name)\n ... };\n {\n default::Villain {\n id: 6c22bdf0...,\n name: 'Sandman',\n name_upper: 'SANDMAN',\n },\n ...\n }\n\nAs with nested filters, the *current scope* changes inside nested shapes.\n\n.. code-block:: edgeql-repl\n\n db> select Villain {\n ... id,\n ... name,\n ... name_upper := str_upper(.name),\n ... nemesis: {\n ... secret_identity,\n ... real_name_upper := str_upper(.secret_identity)\n ... }\n ... };\n {\n default::Villain {\n id: 6c22bdf0...,\n name: 'Sandman',\n name_upper: 'SANDMAN',\n nemesis: default::Hero {\n secret_identity: 'Peter Parker',\n real_name_upper: 'PETER PARKER',\n },\n },\n ...\n }\n\n\n.. _ref_eql_select_backlinks:\n\nBacklinks\n---------\n\n.. api-index:: .<\n\nFetching backlinks is a common use case for computed fields. To demonstrate\nthis, let's fetch a list of all movies starring a particular Hero.\n\n.. code-block:: edgeql-repl\n\n db> select Hero {\n ... name,\n ... movies := . Paths\n `.\n\nInstead of re-declaring backlinks inside every query where they're needed, it's\ncommon to add them directly into your schema as computed links.\n\n.. code-block:: sdl-diff\n\n abstract type Person {\n required name: str {\n constraint exclusive;\n };\n + multi movies := . select Villain {\n ... name,\n ... nemesis_name := .nemesis.name,\n ... movies_with_nemesis := (\n ... select Movie { title }\n ... filter Villain.nemesis in .characters\n ... )\n ... };\n {\n default::Villain {\n name: 'Sandman',\n nemesis_name: 'Spider-Man',\n movies_with_nemesis: {\n default::Movie {title: 'Spider-Man: No Way Home'}\n }\n },\n ...\n }\n\n.. _ref_eql_select_polymorphic:\n\nPolymorphic queries\n-------------------\n\n.. index:: polymorphism\n\nAll queries thus far have referenced concrete object types: ``Hero`` and\n``Villain``. However, both of these types extend the abstract type ``Person``,\nfrom which they inherit the ``name`` property.\n\nPolymorphic sets\n^^^^^^^^^^^^^^^^\n\nIt's possible to directly query all ``Person`` objects; the resulting set will\nbe a mix of ``Hero`` and ``Villain`` objects (and possibly other subtypes of\n``Person``, should they be declared).\n\n.. code-block:: edgeql-repl\n\n db> select Person { name };\n {\n default::Hero {name: 'Spider-Man'},\n default::Hero {name: 'Iron Man'},\n default::Villain {name: 'Doc Ock'},\n default::Villain {name: 'Obadiah Stane'},\n ...\n }\n\nYou may also encounter such \"mixed sets\" when querying a link that points to an\nabstract type (such as ``Movie.characters``) or a :eql:op:`union type\n`.\n\n.. code-block:: edgeql-repl\n\n db> select Movie {\n ... title,\n ... characters: {\n ... name\n ... }\n ... }\n ... filter .title = \"Iron Man 2\";\n {\n default::Movie {\n title: 'Iron Man',\n characters: {\n default::Villain {name: 'Obadiah Stane'},\n default::Hero {name: 'Iron Man'}\n }\n }\n }\n\n\nPolymorphic fields\n^^^^^^^^^^^^^^^^^^\n\n.. api-index:: [is Ā§typeĀ§].\n\nWe can fetch different properties *conditional* on the subtype of each object\nby prefixing property/link references with ``[is ]``. This is known as a\n**polymorphic query**.\n\n.. code-block:: edgeql-repl\n\n db> select Person {\n ... name,\n ... secret_identity := [is Hero].secret_identity,\n ... number_of_villains := count([is Hero].villains),\n ... nemesis := [is Villain].nemesis {\n ... name\n ... }\n ... };\n {\n ...\n default::Villain {\n name: 'Obadiah Stane',\n secret_identity: {},\n number_of_villains: 0,\n nemesis: default::Hero {\n name: 'Iron Man'\n }\n },\n default::Hero {\n name: 'Spider-Man',\n secret_identity: 'Peter Parker',\n number_of_villains: 4,\n nemesis: {}\n },\n ...\n }\n\nThis syntax might look familiar; it's the :ref:`type intersection\n` again. In effect, this operator conditionally\nreturns the value of the referenced field only if the object matches a\nparticular type. If the match fails, an empty set is returned.\n\nThe line ``secret_identity := [is Hero].secret_identity`` is a bit redundant,\nsince the computed property has the same name as the polymorphic field. In\nthese cases, EdgeQL supports a shorthand.\n\n.. code-block:: edgeql-repl\n\n db> select Person {\n ... name,\n ... [is Hero].secret_identity,\n ... [is Villain].nemesis: {\n ... name\n ... }\n ... };\n {\n ...\n default::Villain {\n name: 'Obadiah Stane',\n secret_identity: {},\n nemesis: default::Hero {name: 'Iron Man'}\n },\n default::Hero {\n name: 'Spider-Man',\n secret_identity: 'Peter Parker',\n nemesis: {}\n },\n ...\n }\n\nFiltering polymorphic links\n^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nRelatedly, it's possible to filter polymorphic links by subtype. Below, we\nexclusively fetch the ``Movie.characters`` of type ``Hero``.\n\n.. code-block:: edgeql-repl\n\n db> select Movie {\n ... title,\n ... characters[is Hero]: {\n ... secret_identity\n ... },\n ... };\n {\n default::Movie {\n title: 'Spider-Man: No Way Home',\n characters: {default::Hero {secret_identity: 'Peter Parker'}},\n },\n default::Movie {\n title: 'Iron Man',\n characters: {default::Hero {secret_identity: 'Tony Stark'}},\n },\n ...\n }\n\nAccessing types in polymorphic queries\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nWhile the type of an object is displayed alongside the results of polymorphic\nqueries run in the REPL, this is simply a convenience of the REPL and not a\nproperty that can be accessed. This is particularly noticeable if you cast an\nobject to ``json``, making it impossible to determine the type if the query is\npolymorphic. First, the result of a query as the REPL presents it with type\nannotations displayed:\n\n.. code-block:: edgeql-repl\n\n db> select Person limit 1;\n {default::Villain {id: 6c22bdf0-5c03-11ee-99ff-dfaea4d947ce}}\n\nNote the type ``default::Villain``, which is displayed for the user's\nconvenience but is not actually part of the data returned. This is the same\nquery with the result cast as ``json`` to show only the data returned:\n\n.. code-block:: edgeql-repl\n\n db> select Person limit 1;\n {Json(\"{\\\"id\\\": \\\"6c22bdf0-5c03-11ee-99ff-dfaea4d947ce\\\"}\")}\n\n.. note::\n\n We will continue to cast subesequent examples in this section to ``json``,\n not because this is required for any of the functionality being\n demonstrated, but to remove the convenience type annotations provided by\n the REPL and make it easier to see what data is actually being returned by\n the query.\n\nThe type of an object is found inside ``__type__`` which is a link that\ncarries various information about the object's type, including its ``name``.\n\n.. code-block:: edgeql-repl\n\n db> select Person {\n ... __type__: {\n ... name\n ... }\n ... } limit 1;\n {Json(\"{\\\"__type__\\\": {\\\"name\\\": \\\"default::Villain\\\"}}\")}\n\nThis information can be pulled into the top level by assigning a name to\nthe ``name`` property inside ``__type__``:\n\n.. code-block:: edgeql-repl\n\n db> select Person { type := .__type__.name } limit 1;\n {Json(\"{\\\"type\\\": \\\"default::Villain\\\"}\")}\n\nThere is nothing magical about ``__type__``; it is a simple link to an object\nof the type ``ObjectType`` which contains all of the possible information to\nknow about the type of the current object. The splat operator can be used to\nsee this object's makeup, while the double splat operator produces too much\noutput to show on this page. Playing around with the splat and double splat\noperator inside ``__type__`` is a quick way to get some insight into the\ninternals of Gel.\n\n.. code-block:: edgeql-repl\n\n db> select Person.__type__ {*} limit 1;\n {\n schema::ObjectType {\n id: 48be3a94-5bf3-11ee-bd60-0b44b607e31d,\n name: 'default::Hero',\n internal: false,\n builtin: false,\n computed_fields: [],\n final: false,\n is_final: false,\n abstract: false,\n is_abstract: false,\n inherited_fields: [],\n from_alias: false,\n is_from_alias: false,\n expr: {},\n compound_type: false,\n is_compound_type: false,\n },\n }\n\n.. _ref_eql_select_free_objects:\n\nFree objects\n------------\n\n.. index:: ad hoc type\n\nTo select several values simultaneously, you can \"bundle\" them into a \"free\nobject\". Free objects are a set of key-value pairs that can contain any\nexpression. Here, the term \"free\" is used to indicate that the object in\nquestion is not an instance of a particular *object type*; instead, it's\nconstructed ad hoc inside the query.\n\n.. code-block:: edgeql-repl\n\n db> select {\n ... my_string := \"This is a string\",\n ... my_number := 42,\n ... several_numbers := {1, 2, 3},\n ... all_heroes := Hero { name }\n ... };\n {\n {\n my_string: 'This is a string',\n my_number: 42,\n several_numbers: {1, 2, 3},\n all_heroes: {\n default::Hero {name: 'Spider-Man'},\n default::Hero {name: 'Iron Man'},\n },\n },\n }\n\n\nNote that the result is a *singleton* but each key corresponds to a set of\nvalues, which may have any cardinality.\n\n.. _ref_eql_select_with:\n\nWith block\n----------\n\nAll top-level EdgeQL statements (``select``, ``insert``, ``update``, and\n``delete``) can be prefixed with a ``with`` block. These blocks let you declare\nstandalone expressions that can be used in your query.\n\n.. code-block:: edgeql-repl\n\n db> with hero_name := \"Iron Man\"\n ... select Hero { secret_identity }\n ... filter .name = hero_name;\n {default::Hero {secret_identity: 'Tony Stark'}}\n\n\nFor full documentation on ``with``, see :ref:`EdgeQL > With `.\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`Reference > Commands > Select `\n * - :ref:`Cheatsheets > Selecting data `\n" }, { "path": "docs/reference/edgeql/sets.rst", "content": ".. _ref_eql_sets:\n\nSets\n====\n\n.. _ref_eql_everything_is_a_set:\n\nEverything is a set\n-------------------\n\n.. index:: multiset, cardinality, empty set, singleton\n\nAll values in EdgeQL are actually **sets**: a collection of values of a given\n**type**. All elements of a set must have the same type. The number of items in\na set is known as its **cardinality**. A set with a cardinality of zero is\nreferred to as an **empty set**. A set with a cardinality of one is known as a\n**singleton**.\n\n.. _ref_eql_set_constructor:\n\nConstructing sets\n-----------------\n\n.. api-index:: {Ā§expr [\\, ...]Ā§}, union\n\nSet literals are declared with *set constructor* syntax: a comma-separated\nlist of values inside a set of ``{curly braces}``.\n\n.. code-block:: edgeql-repl\n\n db> select {\"set\", \"of\", \"strings\"};\n {\"set\", \"of\", \"strings\"}\n db> select {1, 2, 3};\n {1, 2, 3}\n\nIn actuality, curly braces are a syntactic sugar for the\n:eql:op:`union` operator. The previous examples are perfectly\nequivalent to the following:\n\n.. code-block:: edgeql-repl\n\n db> select \"set\" union \"of\" union \"strings\";\n {\"set\", \"of\", \"strings\"}\n db> select 1 union 2 union 3;\n {1, 2, 3}\n\nA consequence of this is that nested sets are *flattened*.\n\n.. code-block:: edgeql-repl\n\n db> select {1, {2, {3, 4}}};\n {1, 2, 3, 4}\n db> select 1 union (2 union (3 union 4));\n {1, 2, 3, 4}\n\nAll values in a set must have the same type. For convenience, Gel will\n*implicitly cast* values to other types, as long as there is no loss of\ninformation (e.g. converting a ``int16`` to an ``int64``). For a full\nreference, see the casting table in :ref:`Standard Library > Casts\n`.\n\n.. code-block:: edgeql-repl\n\n db> select {1, 1.5};\n {1.0, 1.5}\n db> select {1, 1234.5678n};\n {1.0n, 1234.5678n}\n\nAttempting to declare a set containing elements of *incompatible* types is not\npermitted.\n\n.. code-block:: edgeql-repl\n\n db> select {\"apple\", 3.14};\n error: QueryError: set constructor has arguments of incompatible types\n 'std::str' and 'std::float64'\n\n.. note::\n\n Types are considered *compatible* if one can be implicitly cast into the\n other. For reference on implicit castability, see :ref:`Standard Library >\n Casts `.\n\n.. _ref_eql_set_literals_are_singletons:\n\nLiterals are singletons\n-----------------------\n\nLiteral syntax like ``6`` or ``\"hello world\"`` is just a shorthand for\ndeclaring a *singleton* of a given type. This is why the literals we created in\nthe previous section were printed inside braces: to indicate that these values\nare *actually sets*.\n\n.. code-block:: edgeql-repl\n\n db> select 6;\n {6}\n db> select \"hello world\";\n {\"hello world\"}\n\nWrapping a literal in curly braces does not change the meaning of the\nexpression. For instance, ``\"hello world\"`` is *exactly equivalent* to\n``{\"hello world\"}``.\n\n.. code-block:: edgeql-repl\n\n db> select {\"hello world\"};\n {\"hello world\"}\n db> select \"hello world\" = {\"hello world\"};\n {true}\n\n\nYou can retrieve the cardinality of a set with the :eql:func:`count` function.\n\n.. code-block:: edgeql-repl\n\n db> select count('aaa');\n {1}\n db> select count({'aaa', 'bbb'});\n {2}\n\n\n.. _ref_eql_empty_sets:\n\nEmpty sets\n----------\n\n.. index:: null, exists\n\nThe reason EdgeQL introduced the concept of *sets* is to eliminate the concept\nof ``null``. In SQL databases ``null`` is a special value denoting the absence\nof data; in Gel the absence of data is just an empty set.\n\n.. note::\n\n Why is the existence of NULL a problem? Put simply, it's an edge case that\n permeates all of SQL and is often handled inconsistently in different\n circumstances. A number of specific inconsistencies are documented in detail\n in the `We Can Do Better Than SQL\n `_\n post on the Gel blog. For broader context, see Tony Hoare's talk\n `\"The Billion Dollar Mistake\" `_.\n\n\nDeclaring empty sets isn't as simple as ``{}``; in EdgeQL, all expressions are\n*strongly typed*, including empty sets. With nonempty sets (like ``{1, 2, 3}``)\n, the type is inferred from the set's contents (``int64``). But with empty sets\nthis isn't possible, so an *explicit cast* is required.\n\n.. code-block:: edgeql-repl\n\n db> select {};\n error: QueryError: expression returns value of indeterminate type\n ā”Œā”€ query:1:8\n ā”‚\n 1 ā”‚ select {};\n ā”‚ ^^ Consider using an explicit type cast.\n\n db> select {};\n {}\n db> select {};\n {}\n db> select count({});\n {0}\n\nYou can check whether or not a set is *empty* with the :eql:op:`exists`\noperator.\n\n.. code-block:: edgeql-repl\n\n db> select exists {};\n {false}\n db> select exists {'not', 'empty'};\n {true}\n\n\n.. _ref_eql_set_references:\n\nSet references\n--------------\n\n.. index:: pointer, alias, with\n\nA set reference is a *pointer* to a set of values. Most commonly, this is the\nname of an :ref:`object type ` you've declared in\nyour schema.\n\n.. code-block:: edgeql-repl\n\n db> select User;\n {\n default::User {id: 9d2ce01c-35e8-11ec-acc3-83b1377efea0},\n default::User {id: b0e0dd0c-35e8-11ec-acc3-abf1752973be},\n }\n db> select count(User);\n {2}\n\nIt may also be an *alias*, which can be defined in a :ref:`with block\n` or as an :ref:`alias declaration ` in your\nschema.\n\n.. note::\n\n In the example above, the ``User`` object type was declared inside the\n ``default`` module. If it was in a non-``default`` module (say,\n ``my_module``, we would need to use its *fully-qualified* name.\n\n .. code-block:: edgeql-repl\n\n db> select my_module::User;\n\n\n.. _ref_eql_set_distinct:\n\nMultisets\n---------\n\n.. api-index:: distinct\n\nTechnically sets in Gel are actually *multisets*, because they can contain\nduplicates of the same element. To eliminate duplicates, use the\n:eql:op:`distinct` set operator.\n\n.. code-block:: edgeql-repl\n\n db> select {'aaa', 'aaa', 'aaa'};\n {'aaa', 'aaa', 'aaa'}\n db> select distinct {'aaa', 'aaa', 'aaa'};\n {'aaa'}\n\n.. _ref_eql_set_in:\n\nChecking membership\n-------------------\n\n.. api-index:: Ā§elementĀ§ in Ā§setĀ§\n\nUse the :eql:op:`in` operator to check whether a set contains a particular\nelement.\n\n.. code-block:: edgeql-repl\n\n db> select 'aaa' in {'aaa', 'bbb', 'ccc'};\n {true}\n db> select 'ddd' in {'aaa', 'bbb', 'ccc'};\n {false}\n\n\n.. _ref_eql_set_union:\n\nMerging sets\n------------\n\n.. api-index:: union\n\nUse the :eql:op:`union` operator to merge two sets.\n\n.. code-block:: edgeql-repl\n\n db> select 'aaa' union 'bbb' union 'ccc';\n {'aaa', 'bbb', 'ccc'}\n db> select {1, 2} union {3.1, 4.4};\n {1.0, 2.0, 3.1, 4.4}\n\nFinding common members\n----------------------\n\n.. api-index:: intersect\n\nUse the :eql:op:`intersect` operator to find common members between two sets.\n\n.. code-block:: edgeql-repl\n\n db> select {1, 2, 3, 4, 5} intersect {3, 4, 5, 6, 7};\n {3, 5, 4}\n db> select {'a', 'b', 'c', 'd', 'e'} intersect {'c', 'd', 'e', 'f', 'g'};\n {'e', 'd', 'c'}\n\nIf set members are repeated in both sets, they will be repeated in the set\nproduced by :eql:op:`intersect` the same number of times they are repeated in\nboth of the operand sets.\n\n.. code-block:: edgeql-repl\n\n db> select {0, 1, 1, 1, 2, 3, 3} intersect {1, 3, 3, 3, 3, 3};\n {1, 3, 3}\n\nIn this example, ``1`` appears three times in the first set but only once in\nthe second, so it appears only once in the result. ``3`` appears twice in the\nfirst set and five times in the second. Both ``3`` appearances in the first set\nare overlapped by ``3`` appearances in the second, so they both end up in the\nresulting set.\n\n\nRemoving common members\n-----------------------\n\n.. api-index:: except\n\nUse the :eql:op:`except` operator to leave only the members in the first set\nthat do not appear in the second set.\n\n.. code-block:: edgeql-repl\n\n db> select {1, 2, 3, 4, 5} except {3, 4, 5, 6, 7};\n {1, 2}\n db> select {'a', 'b', 'c', 'd', 'e'} except {'c', 'd', 'e', 'f', 'g'};\n {'b', 'a'}\n\nWhen :eql:op:`except` eliminates a common member that is repeated, it never\neliminates more than the number of instances of that member appearing in the\nsecond set.\n\n.. code-block:: edgeql-repl\n\n db> select {0, 1, 1, 1, 2, 3, 3} except {1, 3, 3, 3, 3, 3};\n {0, 1, 1, 2}\n\nIn this example, both sets share the member ``1``. The first set contains three\nof them while the second contains only one. The result retains two ``1``\nmembers from the first set since the sets shared only a single ``1`` in common.\nThe second set has five ``3`` members to the first set's two, so both of the\nfirst set's ``3`` members are eliminated from the resulting set.\n\n\n.. _ref_eql_set_coalesce:\n\nCoalescing\n----------\n\n.. index:: empty set, default values, optional\n.. api-index:: ??\n\nOccasionally in queries, you need to handle the case where a set is empty. This\ncan be achieved with a coalescing operator :eql:op:`?? `. This is\ncommonly used to provide default values for optional :ref:`query parameters\n`.\n\n.. code-block:: edgeql-repl\n\n db> select 'value' ?? 'default';\n {'value'}\n db> select {} ?? 'default';\n {'default'}\n\n.. note::\n\n Coalescing is an example of a function/operator with :ref:`optional inputs\n `. By default, passing an empty set into a\n function/operator will \"short circuit\" the operation and return an empty set.\n However it's possible to mark inputs as *optional*, in which case the\n operation will be defined over empty sets. Another example is\n :eql:func:`count`, which returns ``{0}`` when an empty set is passed as\n input.\n\n.. _ref_eql_set_type_filter:\n\nInheritance\n-----------\n\n.. index:: type intersection, backlinks\n.. api-index:: Ā§exprĀ§[is Ā§typeĀ§]\n\n|Gel| schemas support :ref:`inheritance `;\ntypes (usually object types) can extend one or more other types. For instance\nyou may declare an abstract object type ``Media`` that is extended by ``Movie``\nand ``TVShow``.\n\n.. code-block:: sdl\n\n abstract type Media {\n required title: str;\n }\n\n type Movie extending Media {\n release_year: int64;\n }\n\n type TVShow extending Media {\n num_seasons: int64;\n }\n\nA set of type ``Media`` may contain both ``Movie`` and ``TVShow``\nobjects.\n\n.. code-block:: edgeql-repl\n\n db> select Media;\n {\n default::Movie {id: 9d2ce01c-35e8-11ec-acc3-83b1377efea0},\n default::Movie {id: 3bfe4900-3743-11ec-90ee-cb73d2740820},\n default::TVShow {id: b0e0dd0c-35e8-11ec-acc3-abf1752973be},\n }\n\nWe can use the *type intersection* operator ``[is ]`` to restrict the\nelements of a set by subtype.\n\n.. code-block:: edgeql-repl\n\n db> select Media[is Movie];\n {\n default::Movie {id: 9d2ce01c-35e8-11ec-acc3-83b1377efea0},\n default::Movie {id: 3bfe4900-3743-11ec-90ee-cb73d2740820},\n }\n db> select Media[is TVShow];\n {\n default::TVShow {id: b0e0dd0c-35e8-11ec-acc3-abf1752973be}\n }\n\nType filters are commonly used in conjunction with :ref:`backlinks\n`.\n\n.. _ref_eql_set_aggregate:\n\nAggregate vs element-wise operations\n------------------------------------\n\n.. index:: cartesian product\n\nEdgeQL provides a large library of built-in functions and operators for\nhandling data structures. It's useful to consider functions/operators as either\n*aggregate* or *element-wise*.\n\n.. note::\n\n This is an over-simplification, but it's a useful mental model when just\n starting out with Gel. For a more complete guide, see :ref:`Reference >\n Cardinality `.\n\n*Aggregate* operations are applied to the set *as a whole*; they\naccept a set with arbitrary cardinality and return a *singleton* (or perhaps an\nempty set if the input was also empty).\n\n.. code-block:: edgeql-repl\n\n db> select count({'aaa', 'bbb'});\n {2}\n db> select sum({1, 2, 3});\n {6}\n db> select min({1, 2, 3});\n {1}\n\nElement-wise operations are applied on *each element* of a set.\n\n.. code-block:: edgeql-repl\n\n db> select str_upper({'aaa', 'bbb'});\n {'AAA', 'BBB'}\n db> select {1, 2, 3} ^ 2;\n {1, 4, 9}\n db> select str_split({\"hello world\", \"hi again\"}, \" \");\n {[\"hello\", \"world\"], [\"hi\", \"again\"]}\n\nWhen an *element-wise* operation accepts two or more inputs, the operation is\napplied to all possible combinations of inputs; in other words, the operation\nis applied to the *Cartesian product* of the inputs.\n\n.. code-block:: edgeql-repl\n\n db> select {'aaa', 'bbb'} ++ {'ccc', 'ddd'};\n {'aaaccc', 'aaaddd', 'bbbccc', 'bbbddd'}\n\nAccordingly, operations involving an empty set typically return an empty set.\nIn constrast, aggregate operations like :eql:func:`count` are able to operate\non empty sets.\n\n.. code-block:: edgeql-repl\n\n db> select {} ++ 'ccc';\n {}\n db> select count({});\n {0}\n\nFor a more complete discussion of cardinality, see :ref:`Reference >\nCardinality `.\n\n.. _ref_eql_set_array_conversion:\n\nConversion to/from arrays\n-------------------------\n\n.. api-index:: array_unpack, array_agg\n\nBoth arrays and sets are collections of values that share a type. EdgeQL\nprovides ways to convert one into the other.\n\n.. note::\n\n Remember that *all values* in EdgeQL are sets; an array literal is just a\n singleton set of arrays. So here, \"converting\" a set into an array means\n converting a set of type ``x`` into another set with cardinality\n ``1`` (a singleton) and type ``array``.\n\n.. code-block:: edgeql-repl\n\n db> select array_unpack([1,2,3]);\n {1, 2, 3}\n db> select array_agg({1,2,3});\n {[1, 2, 3]}\n\nArrays are an *ordered collection*, whereas sets are generally unordered\n(unless explicitly sorted with an ``order by`` clause in a :ref:`select\n` statement).\n\nElement-wise scalar operations in the standard library cannot be applied to\narrays, so sets of scalars are typically easier to manipulate, search, and\ntransform than arrays.\n\n.. code-block:: edgeql-repl\n\n db> select str_trim({' hello', 'world '});\n {'hello', 'world'}\n db> select str_trim([' hello', 'world ']);\n error: QueryError: function \"str_trim(arg0: array)\" does not exist\n\nSome :ref:`aggregate ` operations have\nanalogs that operate on arrays. For instance, the set function\n:eql:func:`count` is analogous to the array function :eql:func:`len`.\n\n\nReference\n---------\n\n.. list-table::\n\n * - Set operators\n - :eql:op:`distinct` :eql:op:`in` :eql:op:`union`\n :eql:op:`exists` :eql:op:`if..else`\n :eql:op:`?? ` :eql:op:`detached`\n :eql:op:`[is type] `\n * - Utility functions\n - :eql:func:`count` :eql:func:`enumerate`\n * - Cardinality assertion\n - :eql:func:`assert_distinct` :eql:func:`assert_single`\n :eql:func:`assert_exists`\n" }, { "path": "docs/reference/edgeql/transactions.rst", "content": ".. _ref_eql_transactions:\n\nTransactions\n============\n\n.. api-index:: start transaction, declare savepoint, release savepoint,\n rollback to savepoint, rollback, commit\n\nEdgeQL supports atomic transactions. The transaction API consists\nof several commands:\n\n:eql:stmt:`start transaction`\n Start a transaction, specifying the isolation level, access mode (``read\n only`` vs ``read write``), and deferrability.\n\n:eql:stmt:`declare savepoint`\n Establish a new savepoint within the current transaction. A savepoint is a\n intermediate point in a transaction flow that provides the ability to\n partially rollback a transaction.\n\n:eql:stmt:`release savepoint`\n Destroys a savepoint previously defined in the current transaction.\n\n:eql:stmt:`rollback to savepoint`\n Rollback to the named savepoint. All changes made after the savepoint\n are discarded. The savepoint remains valid and can be rolled back\n to again later, if needed.\n\n:eql:stmt:`rollback`\n Rollback the entire transaction. All updates made within the transaction are\n discarded.\n\n:eql:stmt:`commit`\n Commit the transaction. All changes made by the transaction become visible\n to others and will persist if a crash occurs.\n\n\nClient libraries\n----------------\n\nThere is rarely a reason to use these commands directly. All Gel client\nlibraries provide dedicated transaction APIs that handle transaction creation\nunder the hood.\n\nExamples below show a transaction that sends 10 cents from the account\nof a ``BankCustomer`` called ``'Customer1'`` to ``BankCustomer`` called\n``'Customer2'``. The equivalent Gel schema and queries are:\n\n.. code-block::\n\n module default {\n type BankCustomer {\n required name: str;\n required balance: int64;\n }\n }\n update BankCustomer\n filter .name = 'Customer1'\n set { bank_balance := .bank_balance -10 };\n update BankCustomer\n filter .name = 'Customer2'\n set { bank_balance := .bank_balance +10 }\n\nTypeScript/JS\n^^^^^^^^^^^^^\n\nUsing an EdgeQL query string:\n\n.. code-block:: typescript\n\n client.transaction(async tx => {\n await tx.execute(`update BankCustomer\n filter .name = 'Customer1'\n set { bank_balance := .bank_balance -10 }`);\n await tx.execute(`update BankCustomer\n filter .name = 'Customer2'\n set { bank_balance := .bank_balance +10 }`);\n });\n\nUsing the querybuilder:\n\n.. code-block:: typescript\n\n const query1 = e.update(e.BankCustomer, () => ({\n filter_single: { name: \"Customer1\" },\n set: {\n bank_balance: { \"-=\": 10 }\n },\n }));\n const query2 = e.update(e.BankCustomer, () => ({\n filter_single: { name: \"Customer2\" },\n set: {\n bank_balance: { \"+=\": 10 }\n },\n }));\n\n client.transaction(async (tx) => {\n await query1.run(tx);\n await query2.run(tx);\n });\n\nFull documentation at :ref:`Client Libraries > TypeScript/JS `;\n\nPython\n^^^^^^\n\n.. code-block:: python\n\n async for tx in client.transaction():\n async with tx:\n await tx.execute(\"\"\"update BankCustomer\n filter .name = 'Customer1'\n set { bank_balance := .bank_balance -10 };\"\"\")\n await tx.execute(\"\"\"update BankCustomer\n filter .name = 'Customer2'\n set { bank_balance := .bank_balance +10 };\"\"\")\n\nFull documentation at :ref:`Client Libraries > Python `;\n\nGolang\n^^^^^^\n\n.. code-block:: go\n\n\terr = client.Tx(ctx, func(ctx context.Context, tx *gel.Tx) error {\n\t\tquery1 := `update BankCustomer\n\t\t\tfilter .name = 'Customer1'\n\t\t\tset { bank_balance := .bank_balance -10 };`\n\t\tif e := tx.Execute(ctx, query1); e != nil {\n\t\t\treturn e\n\t\t}\n\t\tquery2 := `update BankCustomer\n\t\t\tfilter .name = 'Customer2'\n\t\t\tset { bank_balance := .bank_balance +10 };`\n\t\tif e := tx.Execute(ctx, query2); e != nil {\n\t\t\treturn e\n\t\t}\n\t\treturn nil\n\t})\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\nFull documentation at `Client Libraries > Go `_.\n\nRust\n^^^^\n\n.. code-block:: rust\n\n let balance_change_query = \"update BankCustomer\n filter .name = $0\n set { bank_balance := .bank_balance + $1 }\";\n\n client\n .transaction(|mut conn| async move {\n conn.execute(balance_change_query, &(\"Customer1\", -10))\n .await\n .expect(\"Execute should have worked\");\n conn.execute(balance_change_query, &(\"Customer2\", 10))\n .await\n .expect(\"Execute should have worked\");\n Ok(())\n })\n .await\n .expect(\"Transaction should have worked\");\n\n.. XXX: Add Rust docs\n.. Full documentation at :ref:`Client Libraries > Rust `.\n\n.. _gel-go: https://pkg.go.dev/github.com/geldata/gel-go" }, { "path": "docs/reference/edgeql/types.rst", "content": ".. _ref_eql_types:\n\n\n=====\nTypes\n=====\n\nThe foundation of EdgeQL is Gel's rigorous type system. There is a set of\nEdgeQL operators and functions for changing, introspecting, and filtering by\ntypes.\n\n.. _ref_eql_types_names:\n\nType expressions\n----------------\n\n.. api-index:: array<Ā§typeĀ§>, tuple<Ā§type [\\, ...]Ā§>\n\nType expressions are exactly what they sound like: EdgeQL expressions that\nrefer to a type. Most commonly, these are simply the *names* of established\ntypes: ``str``, ``int64``, ``BlogPost``, etc. Arrays and tuples have a\ndedicated type syntax.\n\n.. list-table::\n\n * - **Type**\n - **Syntax**\n * - Array\n - ``array``\n * - Tuple (unnamed)\n - ``tuple``\n * - Tuple (named)\n - ``tuple``\n\nFor additional details on type syntax, see :ref:`Schema > Primitive Types\n`.\n\n.. _ref_eql_types_typecast:\n\nType casting\n------------\n\n.. index:: casts, find object by id\n.. api-index:: <Ā§typeĀ§>Ā§exprĀ§\n\nType casting is used to convert primitive values into another type. Casts are\nindicated with angle brackets containing a type expression.\n\n.. code-block:: edgeql-repl\n\n db> select 10;\n {\"10\"}\n db> select 10;\n {10n}\n db> select >[1, 2, 3];\n {['1', '2', '3']}\n db> select >(1, 2, 3);\n {('1', 2, 3n)}\n\n\n\nType casts are useful for declaring literals for types like ``datetime``,\n``uuid``, and ``int16`` that don't have a dedicated syntax.\n\n.. code-block:: edgeql-repl\n\n db> select '1999-03-31T15:17:00Z';\n {'1999-03-31T15:17:00Z'}\n db> select 42;\n {42}\n db> select '89381587-705d-458f-b837-860822e1b219';\n {89381587-705d-458f-b837-860822e1b219}\n\n\nThere are limits to what values can be cast to a certain type. In some cases\ntwo types are entirely incompatible, like ``bool`` and ``int64``; in other\ncases, the source data must be in a particular format, like casting ``str`` to\n``datetime``. For a comprehensive table of castability, see :ref:`Standard\nLibrary > Casts `.\n\nType casts can only be used on primitive expressions, not object type\nexpressions. Every object stored in the database is strongly and immutably\ntyped; you can't simply convert an object to an object of a different type.\n\n.. code-block:: edgeql-repl\n\n db> select 10;\n QueryError: cannot cast 'std::int64' to 'default::BlogPost'\n db> select 'asdf';\n InvalidValueError: invalid input syntax for type std::int64: \"asdf\"\n db> select 100000000000000n;\n NumericOutOfRangeError: std::int16 out of range\n\n\n.. lint-off\n\nYou can cast a UUID into an object:\n\n.. code-block:: edgeql-repl\n\n db> select '01d9cc22-b776-11ed-8bef-73f84c7e91e7';\n {default::Hero {id: 01d9cc22-b776-11ed-8bef-73f84c7e91e7}}\n\nIf you try to cast a UUID that no object of the type has as its ``id``\nproperty, you'll get an error:\n\n.. code-block:: edgeql-repl\n\n db> select 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa';\n gel error: CardinalityViolationError: 'default::Hero' with id 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa' does not exist\n\n.. lint-on\n\n\n.. _ref_eql_types_intersection:\n\nType intersections\n------------------\n\n.. api-index:: [is Ā§typeĀ§]\n\nAll elements of a given set have the same type; however, in the context of\n*sets of objects*, this type might be ``abstract`` and contain elements of\nmultiple concrete subtypes. For instance, a set of ``Media`` objects may\ncontain both ``Movie`` and ``TVShow`` objects.\n\n.. code-block:: edgeql-repl\n\n db> select Media;\n {\n default::Movie {id: 9d2ce01c-35e8-11ec-acc3-83b1377efea0},\n default::Movie {id: 3bfe4900-3743-11ec-90ee-cb73d2740820},\n default::TVShow {id: b0e0dd0c-35e8-11ec-acc3-abf1752973be},\n }\n\nWe can use the *type intersection* operator to restrict the elements of a set\nby subtype.\n\n.. code-block:: edgeql-repl\n\n db> select Media[is Movie];\n {\n default::Movie {id: 9d2ce01c-35e8-11ec-acc3-83b1377efea0},\n default::Movie {id: 3bfe4900-3743-11ec-90ee-cb73d2740820},\n }\n\nLogically, this computes the intersection of the ``Media`` and ``Movie`` sets;\nsince only ``Movie`` objects occur in both sets, this can be conceptualized as\na \"filter\" that removes all elements that aren't of type ``Movie``.\n\n.. Type unions\n.. -----------\n\n.. You can create a type union with the pipe operator: :eql:op:`type | type\n.. `. This is mostly commonly used for object types.\n\n.. .. code-block:: edgeql-repl\n\n.. db> select 5 is int32 | int64;\n.. {true}\n.. db> select Media is Movie | TVShow;\n.. {true, true, true}\n\n\nType checking\n-------------\n\n.. api-index:: Ā§exprĀ§ is Ā§typeĀ§, Ā§exprĀ§ is not Ā§typeĀ§\n\nThe ``[is foo]`` \"type intersection\" syntax should not be confused with the\n*type checking* operator :eql:op:`is`.\n\n.. code-block:: edgeql-repl\n\n db> select 5 is int64;\n {true}\n db> select {3.14, 2.718} is not int64;\n {true, true}\n db> select Media is Movie;\n {true, true, false}\n\n\nThe ``typeof`` operator\n-----------------------\n\n.. api-index:: typeof Ā§exprĀ§\n\nThe type of any expression can be extracted with the :eql:op:`typeof`\noperator. This can be used in any expression that expects a type.\n\n.. code-block:: edgeql-repl\n\n db> select '100';\n {100}\n db> select \"tuna\" is typeof \"trout\";\n {true}\n\nIntrospection\n-------------\n\nThe entire type system of Gel is *stored inside Gel*. All types are\nintrospectable as instances of the ``schema::Type`` type. For a set of\nintrospection examples, see :ref:`Guides > Introspection\n`.\n" }, { "path": "docs/reference/edgeql/update.rst", "content": ".. _ref_eql_update:\n\nUpdate\n======\n\n.. api-index:: update, filter, set\n\nThe ``update`` command is used to update existing objects.\n\n.. code-block:: edgeql-repl\n\n db> update Hero\n ... filter .name = \"Hawkeye\"\n ... set { name := \"Ronin\" };\n {default::Hero {id: d476b12e-3e7b-11ec-af13-2717f3dc1d8a}}\n\nIf you omit the ``filter`` clause, all objects will be updated. This is useful\nfor updating values across all objects of a given type. The example below\ncleans up all ``Hero.name`` values by trimming whitespace and converting them\nto title case.\n\n.. code-block:: edgeql-repl\n\n db> update Hero\n ... set { name := str_trim(str_title(.name)) };\n {default::Hero {id: d476b12e-3e7b-11ec-af13-2717f3dc1d8a}}\n\nSyntax\n^^^^^^\n\nThe structure of the ``update`` statement (``update...filter...set``) is an\nintentional inversion of SQL's ``UPDATE...SET...WHERE`` syntax. Curiously, in\nSQL, the ``where`` clauses typically occur *last* despite being applied before\nthe ``set`` statement. EdgeQL is structured to reflect this; first, a target\nset is specified, then filters are applied, then the data is updated.\n\nUpdating properties\n-------------------\n\nTo explicitly unset a property that is not required, set it to an empty set.\n\n.. code-block:: edgeql\n\n update Person filter .id = $id set { middle_name := {} };\n\nUpdating links\n--------------\n\n.. api-index:: :=, +=, -=\n\nWhen updating links, the ``:=`` operator will *replace* the set of linked\nvalues.\n\n.. code-block:: edgeql-repl\n\n db> update movie\n ... filter .title = \"Black Widow\"\n ... set {\n ... characters := (\n ... select Person\n ... filter .name in { \"Black Widow\", \"Yelena\", \"Dreykov\" }\n ... )\n ... };\n {default::Title {id: af706c7c-3e98-11ec-abb3-4bbf3f18a61a}}\n db> select Movie { num_characters := count(.characters) }\n ... filter .title = \"Black Widow\";\n {default::Movie {num_characters: 3}}\n\nTo add additional linked items, use the ``+=`` operator.\n\n.. code-block:: edgeql-repl\n\n db> update Movie\n ... filter .title = \"Black Widow\"\n ... set {\n ... characters += (insert Villain {name := \"Taskmaster\"})\n ... };\n {default::Title {id: af706c7c-3e98-11ec-abb3-4bbf3f18a61a}}\n db> select Movie { num_characters := count(.characters) }\n ... filter .title = \"Black Widow\";\n {default::Movie {num_characters: 4}}\n\nTo remove items, use ``-=``.\n\n.. code-block:: edgeql-repl\n\n db> update Movie\n ... filter .title = \"Black Widow\"\n ... set {\n ... characters -= Villain # remove all villains\n ... };\n {default::Title {id: af706c7c-3e98-11ec-abb3-4bbf3f18a61a}}\n db> select Movie { num_characters := count(.characters) }\n ... filter .title = \"Black Widow\";\n {default::Movie {num_characters: 2}}\n\nReturning data on update\n------------------------\n\nBy default, ``update`` returns only the inserted object's ``id`` as seen in the\nexamples above. If you want to get additional data back, you may wrap your\n``update`` with a ``select`` and apply a shape specifying any properties and\nlinks you want returned:\n\n.. code-block:: edgeql-repl\n\n db> select (update Hero\n ... filter .name = \"Hawkeye\"\n ... set { name := \"Ronin\" }\n ... ) {id, name};\n {\n default::Hero {\n id: d476b12e-3e7b-11ec-af13-2717f3dc1d8a,\n name: \"Ronin\"\n }\n }\n\nWith blocks\n-----------\n\nAll top-level EdgeQL statements (``select``, ``insert``, ``update``, and\n``delete``) can be prefixed with a ``with`` block. This is useful for updating\nthe results of a complex query.\n\n.. code-block:: edgeql-repl\n\n db> with people := (\n ... select Person\n ... order by .name\n ... offset 3\n ... limit 3\n ... )\n ... update people\n ... set { name := str_trim(.name) };\n {\n default::Hero {id: d4764c66-3e7b-11ec-af13-df1ba5b91187},\n default::Hero {id: d7d7e0f6-40ae-11ec-87b1-3f06bed494b9},\n default::Villain {id: d477a836-3e7b-11ec-af13-4fea611d1c31},\n }\n\n.. note::\n\n You can pass any object-type expression into ``update``, including\n polymorphic ones (as above).\n\nYou can also use ``with`` to make returning additional data from an update more\nreadable:\n\n.. code-block:: edgeql-repl\n\n db> with UpdatedHero := (update Hero\n ... filter .name = \"Hawkeye\"\n ... set { name := \"Ronin\" }\n ... )\n ... select UpdatedHero {\n ... id,\n ... name\n ... };\n {\n default::Hero {\n id: d476b12e-3e7b-11ec-af13-2717f3dc1d8a,\n name: \"Ronin\"\n }\n }\n\n\nSee also\n--------\n\nFor documentation on performing *upsert* operations, see :ref:`EdgeQL > Insert\n> Upserts `.\n\n.. list-table::\n\n * - :ref:`Reference > Commands > Update `\n * - :ref:`Cheatsheets > Updating data `\n" }, { "path": "docs/reference/edgeql/with.rst", "content": ".. _ref_eql_with:\n\nWith\n====\n\n.. index:: composition, composing queries, composable, CTE,\n common table expressions, subquery, subqueries\n.. api-index:: with\n\nAll top-level EdgeQL statements (``select``, ``insert``, ``update``, and\n``delete``) can be prefixed by a ``with`` block. These blocks contain\ndeclarations of standalone expressions that can be used in your query.\n\n.. code-block:: edgeql-repl\n\n db> with my_str := \"hello world\"\n ... select str_title(my_str);\n {'Hello World'}\n\n\nThe ``with`` clause can contain more than one variable. Earlier variables can\nbe referenced by later ones. Taken together, it becomes possible to write\n\"script-like\" queries that execute several statements in sequence.\n\n.. code-block:: edgeql-repl\n\n db> with a := 5,\n ... b := 2,\n ... c := a ^ b\n ... select c;\n {25}\n\n\nSubqueries\n^^^^^^^^^^\n\nThere's no limit to the complexity of computed expressions. EdgeQL is fully\ncomposable; queries can simply be embedded inside each other. The following\nquery fetches a list of all movies featuring at least one of the original six\nAvengers.\n\n.. code-block:: edgeql-repl\n\n db> with avengers := (select Hero filter .name in {\n ... 'Iron Man',\n ... 'Black Widow',\n ... 'Captain America',\n ... 'Thor',\n ... 'Hawkeye',\n ... 'The Hulk'\n ... })\n ... select Movie {title}\n ... filter avengers in .characters;\n {\n\n default::Movie {title: 'Iron Man'},\n default::Movie {title: 'The Incredible Hulk'},\n default::Movie {title: 'Iron Man 2'},\n default::Movie {title: 'Thor'},\n default::Movie {title: 'Captain America: The First Avenger'},\n ...\n }\n\n.. _ref_eql_with_params:\n\nQuery parameters\n^^^^^^^^^^^^^^^^\n\nA common use case for ``with`` clauses is the initialization of :ref:`query\nparameters `.\n\n.. code-block:: edgeql\n\n with user_id := $user_id\n select User { name }\n filter .id = user_id;\n\nFor a full reference on using query parameters, see :ref:`EdgeQL > Parameters\n`.\n\n\nModule alias\n^^^^^^^^^^^^\n\n.. api-index:: with, as module\n\nAnother use of ``with`` is to provide aliases for modules. This can be useful\nfor long queries which reuse many objects or functions from the same module.\n\n.. code-block:: edgeql\n\n with http as module std::net::http\n select http::ScheduledRequest\n filter .method = http::Method.POST;\n\nIf the aliased module does not exist at the top level, but does exists as a\npart of the ``std`` module, that will be used automatically.\n\n.. code-block:: edgeql\n\n with http as module net::http # <- omitting std\n select http::ScheduledRequest\n filter .method = http::Method.POST;\n\n\nModule selection\n^^^^^^^^^^^^^^^^\n\n.. index:: fully-qualified names\n.. api-index:: with module\n\nBy default, the *active module* is ``default``, so all schema objects inside\nthis module can be referenced by their *short name*, e.g. ``User``,\n``BlogPost``, etc. To reference objects in other modules, we must use\nfully-qualified names (``default::Hero``).\n\nHowever, ``with`` clauses also provide a mechanism for changing the *active\nmodule* on a per-query basis.\n\n.. code-block:: edgeql-repl\n\n db> with module schema\n ... select ObjectType;\n\nThis ``with module`` clause changes the default module to schema, so we can\nrefer to ``schema::ObjectType`` (a built-in Gel type) as simply\n``ObjectType``.\n\nAs with module aliases, if the active module does not exist at the top level,\nbut does exist as part of the ``std`` module, that will be used automatically.\n\n.. code-block:: edgeql-repl\n\n db> with module math select abs(-1);\n {1}\n\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`Reference > Commands > With `\n" }, { "path": "docs/reference/index.rst", "content": "=========\nReference\n=========\n\n.. toctree::\n :maxdepth: 3\n :hidden:\n\n using/index\n running/index\n datamodel/index\n edgeql/index\n stdlib/index\n ai/index\n auth/index\n reference/index\n\nLearn three components, and you know |Gel|: how to work with\n:ref:`schema `, how to write queries with\n:ref:`EdgeQL `, and what's available to you in our\n:ref:`standard library `. Start in those sections if you're new to |Gel|.\nMove over to our :ref:`reference ` when you're ready to\ndive deep into the internals, syntax, and other advanced topics.\n\n\nSchema\n------\n\n|Gel| schemas are declared using our schema definition language (SDL).\n\n.. code-block:: sdl\n\n module default {\n type Book {\n required title: str;\n release_year: int16;\n author: Person;\n }\n type Person {\n required name: str;\n }\n }\n\nThe example schema above defines two types: Book and Person, each with\na property or two. Book also contains a link to the author, which is a\nlink to objects of the Person type. Learn more about how to define\nyour schema using SDL in the :ref:`schema ` section.\n\nEdgeQL\n------\n\nEdgeQL is a next-generation query language designed to match SQL in power and\nsurpass it in terms of clarity, brevity, and intuitiveness.\n\n.. code-block:: edgeql-repl\n\n db> select Book {\n ... title,\n ... release_year,\n ... author: {\n ... name\n ... }\n ... } order by .title;\n {\n default::Book {\n title: '1984',\n release_year: 1949,\n author: default::Person {\n name: 'George Orwell'\n }\n },\n default::Book {\n title: 'Americanah',\n release_year: 2013,\n author: default::Person {\n name: 'Chimamanda Ngozi Adichie'\n }\n },\n ...\n }\n\nYou can use EdgeQL to easily return nested data structures just by putting a\nshape with a link on an object as shown above.\n\n\nStandard library\n----------------\n\n|Gel| comes with a rigorously defined type system consisting of scalar\ntypes, collection types (like arrays and tuples), and object types. It\nalso includes a library of built-in functions and operators for working\nwith each datatype, alongside some additional utilities and extensions.\n\n.. code-block:: edgeql-repl\n\n db> select count(Book);\n {16}\n db> select Book {\n ... title,\n ... title_length := len(.title)\n ... } order by .title_length;\n {\n default::Book {\n title: 'Sula',\n title_length: 4\n },\n default::Book {\n title: '1984',\n title_length: 4\n },\n default::Book {\n title: 'Beloved',\n title_length: 7\n },\n default::Book {\n title: 'The Fellowship of the Ring',\n title_length: 26\n },\n default::Book {\n title: 'One Hundred Years of Solitude',\n title_length: 29\n },\n }\n db> select math::stddev(len(Book.title));\n {7.298401651503339}\n\nGel comes with a rigorously defined type system consisting of scalar\ntypes, collection types (like arrays and tuples), and object types. It\nalso includes a library of built-in functions and operators for working\nwith each datatype, alongside some additional utilities and extensions.\n\n\nCheatsheets\n-----------\n\nLearn to do various common tasks using the many tools included with |Gel|.\n\nQuerying\n^^^^^^^^\n\n* :ref:`Select `\n* :ref:`Insert `\n* :ref:`Update `\n* :ref:`Delete `\n* :ref:`via GraphQL `\n\nSchema\n^^^^^^\n\n* :ref:`Booleans `\n* :ref:`Object Types `\n* :ref:`Functions `\n* :ref:`Aliases `\n* :ref:`Annotations `\n* :ref:`Link Properties `\n\nAdmin\n^^^^^\n* :ref:`CLI `\n* :ref:`REPL `\n* :ref:`Admin `\n" }, { "path": "docs/reference/reference/edgeql/analyze.rst", "content": ".. _ref_eql_statements_analyze:\n\nAnalyze\n=======\n\n:eql-statement:\n\n``analyze`` -- trigger performance analysis of the appended query\n\n.. eql:synopsis::\n\n analyze ;\n\n # where is any EdgeQL query\n\n\nDescription\n-----------\n\n``analyze`` returns a table with performance metrics broken down by node.\n\nYou may prepend the ``analyze`` keyword in either of our REPLs (CLI or :ref:`UI\n`) or you may prepend in the UI's query builder for a\nhelpful visualization of your query's performance.\n\nAfter any ``analyze`` in a REPL, run the ``\\expand`` command to see\nfine-grained performance analysis of the previously analyzed query.\n\n\nExample\n-------\n\n.. code-block:: edgeql-repl\n\n db> analyze select Hero {\n ... name,\n ... secret_identity,\n ... villains: {\n ... name,\n ... nemesis: {\n ... name\n ... }\n ... }\n ... };\n ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ Query ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€\n analyze select āžŠ Hero {name, secret_identity, āž‹ villains: {name, āžŒ nemesis: {name}}};\n\n ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ Coarse-grained Query Plan ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€\n ā”‚ Time Cost Loops Rows Width ā”‚ Relations\n āžŠ root ā”‚ 0.0 69709.48 1.0 0.0 32 ā”‚ Hero\n ā•°ā”€ā”€āž‹ .villains ā”‚ 0.0 92.9 0.0 0.0 32 ā”‚ Villain, Hero.villains\n ā•°ā”€ā”€āžŒ .nemesis ā”‚ 0.0 8.18 0.0 0.0 32 ā”‚ Hero\n\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`CLI > gel analyze `\n * - :ref:`EdgeQL > Analyze `\n" }, { "path": "docs/reference/reference/edgeql/cardinality.rst", "content": ".. _ref_reference_cardinality:\n\nCardinality\n===========\n\n\nThe number of items in a set is known as its **cardinality**. A set with a\ncardinality of zero is referred to as an **empty set**. A set with a\ncardinality of one is known as a **singleton**.\n\nTerminology\n-----------\n\nThe term **cardinality** is used to refer to both the *exact* number of\nelements in a given set or a *range* of possible values. Internally, Gel\ntracks 5 different cardinality ranges: ``Empty`` (zero elements), ``One`` (a\nsingleton set), ``AtMostOne`` (zero or one elements), ``AtLeastOne`` (one or\nmore elements), and ``Many`` (any number of elements).\n\n|Gel| uses this information to statically check queries for validity. For\ninstance, when assigning to a ``required multi`` link, the value being\nassigned in question *must* have a cardinality of ``One`` or ``AtLeastOne``\n(as empty sets are not permitted).\n\n.. _ref_reference_cardinality_functions_operators:\n\nFunctions and operators\n-----------------------\n\nIt's often useful to think of Gel functions/operators as either\n*element-wise* or *aggregate*. Element-wise operations are applied to *each\nitem* in a set. Aggregate operations operate on sets *as a whole*.\n\n.. note::\n\n This is a simplification, but it's a useful mental model when getting\n started with Gel.\n\n.. _ref_reference_cardinality_aggregate:\n\nAggregate operations\n^^^^^^^^^^^^^^^^^^^^\n\nAn example of an aggregate function is :eql:func:`count`. It returns the number\nof elements in a given set. Regardless of the size of the input set, the result\nis a singleton integer.\n\n.. code-block:: edgeql-repl\n\n db> select count('hello');\n {1}\n db> select count({'this', 'is', 'a', 'set'});\n {4}\n db> select count({});\n {0}\n\nAnother example is :eql:func:`array_agg`, which converts a *set* of elements\ninto a singleton array.\n\n.. code-block:: edgeql-repl\n\n db> select array_agg({1,2,3});\n {[1, 2, 3]}\n\n\n.. _ref_reference_cardinality_elementwise:\n\nElement-wise operations\n^^^^^^^^^^^^^^^^^^^^^^^\n\nBy contrast, the :eql:func:`len` function is element-wise; it computes the\nlength of each string inside a set of strings; as such, it converts a set\nof :eql:type:`str` into an equally-sized set of :eql:type:`int64`.\n\n.. code-block:: edgeql-repl\n\n db> select len('hello');\n {5}\n db> select len({'hello', 'world'});\n {5, 5}\n\n.. _ref_reference_cardinality_cartesian:\n\nCartesian products\n^^^^^^^^^^^^^^^^^^\n\nIn case of element-wise operations that accept multiple arguments, the\noperation is applied to a cartesian product of all the input sets.\n\n.. code-block:: edgeql-repl\n\n db> select {'aaa', 'bbb'} ++ {'ccc', 'ddd'};\n {'aaaccc', 'aaaddd', 'bbbccc', 'bbbddd'}\n db> select {true, false} or {true, false};\n {true, true, true, false}\n\nBy extension, if any of the input sets are empty, the result of applying an\nelement-wise function is also empty. In effect, when Gel detects an empty\nset, it \"short-circuits\" and returns an empty set without applying the\noperation.\n\n.. code-block:: edgeql-repl\n\n db> select {} ++ {'ccc', 'ddd'};\n {}\n db> select {} or {true, false};\n {}\n\n.. note::\n\n Certain functions and operators avoid this \"short-circuit\" behavior by\n marking their inputs as :ref:`optional `. A\n notable example of an operator with optional inputs is the :eql:op:`??\n ` operator.\n\n .. code-block:: edgeql-repl\n\n db> select {} ?? 'default';\n {'default'}\n\n\nPer-input cardinality\n^^^^^^^^^^^^^^^^^^^^^\n\nUltimately, the distinction between \"aggregate vs element-wise\" operations is\na false one. Consider the :eql:op:`in` operation.\n\n.. code-block:: edgeql-repl\n\n db> select {1, 4} in {1, 2, 3};\n {true, false}\n\nThis operator takes two inputs. If it was \"element-wise\" we would expect the\ncardinality of the above operation to the cartesian product of the input\ncardinalities: ``2 x 3 = 6``. It it was aggregate, we'd expect a singleton\noutput.\n\nInstead, the cardinality is ``2``. This operator is element-wise with respect\nto the first input and aggregate with respect to the second. The \"element-wise\nvs aggregate\" concept isn't determined on a per-function/per-operator basis;\nit determined on a per-input basis.\n\n\nType qualifiers\n^^^^^^^^^^^^^^^\n\nWhen defining functions, all inputs are element-wise by default. The\n``set of`` :ref:`type qualifier ` is used to\ndesignate an input as *aggregate*. Currently this modifier is not supported\nfor user-defined functions, but it is used by certain standard library\nfunctions.\n\nSimilarly the ``optional`` qualifier marks the input as optional; an operation\nwill be executed is an optional input is empty, whereas passing an\nempty set for a \"standard\" (non-optional) element-wise input will always\nresult in an empty set.\n\nSimilarly, the *output* of a function :ref:`can be annotated\n` with ``set of`` and ``optional`` qualifiers.\n\n\nCardinality computation\n^^^^^^^^^^^^^^^^^^^^^^^\n\nTo compute the number of times a function/operator will be invoked, take the\ncardinality of each input and apply the following transformations, based on\nthe type qualifier (or lack thereof) for each:\n\n.. code-block::\n\n element-wise: N -> N\n optional: N -> max(1, N)\n aggregate: N -> 1\n\nThe ultimate cardinality of the result is the union of the results of each\ninvokation; as such, it depends on the *values returned* by each invokation.\n" }, { "path": "docs/reference/reference/edgeql/casts.csv", "content": "from \\ to,:eql:type:`json `,:eql:type:`str `,:eql:type:`float32 `,:eql:type:`float64 `,:eql:type:`int16 `,:eql:type:`int32 `,:eql:type:`int64 `,:eql:type:`bigint `,:eql:type:`decimal `,:eql:type:`bool `,:eql:type:`bytes `,:eql:type:`uuid `,:eql:type:`datetime `,:eql:type:`duration `,:eql:type:`local_date `,:eql:type:`local_datetime `,:eql:type:`local_time `,:eql:type:`relative_duration `,:eql:type:`date_duration `,:eql:type:`enum`,object\n:eql:type:`json `,,``<>``,``<>``,``<>``,``<>``,``<>``,``<>``,``<>``,``<>``,``<>``,``<>``,``<>``,``<>``,``<>``,``<>``,``<>``,``<>``,``<>``,``<>``,``<>``,\n:eql:type:`str `,``<>``,,``<>``,``<>``,``<>``,``<>``,``<>``,``<>``,``<>``,``<>``,,``<>``,``<>``,``<>``,``<>``,``<>``,``<>``,``<>``,``<>``,``:=``,\n:eql:type:`float32 `,``<>``,``<>``,,impl,``<>*``,``<>*``,``<>*``,``<>*``,``<>``,,,,,,,,,,,,\n:eql:type:`float64 `,``<>``,``<>``,``:=``,,``<>*``,``<>*``,``<>*``,``<>*``,``<>``,,,,,,,,,,,,\n:eql:type:`int16 `,``<>``,``<>``,impl,impl,,impl,impl,impl,impl,,,,,,,,,,,,\n:eql:type:`int32 `,``<>``,``<>``,,impl,``<>``,,impl,impl,impl,,,,,,,,,,,,\n:eql:type:`int64 `,``<>``,``<>``,``:=``,impl,``:=``,``:=``,,impl,impl,,,,,,,,,,,,\n:eql:type:`bigint `,,,,,,,,,impl,,,,,,,,,,,,\n:eql:type:`decimal `,``<>``,``<>``,``<>``,``<>``,``<>``,``<>``,``<>``,``<>``,,,,,,,,,,,,,\n:eql:type:`bool `,``<>``,``<>``,,,,,,,,,,,,,,,,,,,\n:eql:type:`bytes `,``<>``,,,,,,,,,,,,,,,,,,,,\n:eql:type:`uuid `,``<>``,``<>``,,,,,,,,,,,,,,,,,,,``<>``\n:eql:type:`datetime `,``<>``,``<>``,,,,,,,,,,,,,,,,,,,\n:eql:type:`duration `,``<>``,``<>``,,,,,,,,,,,,,,,,``<>``,,,\n:eql:type:`local_date `,``<>``,``<>``,,,,,,,,,,,,,,impl,,,,,\n:eql:type:`local_datetime `,``<>``,``<>``,,,,,,,,,,,,,``<>``,,``<>``,,,,\n:eql:type:`local_time `,``<>``,``<>``,,,,,,,,,,,,,,,,,,,\n:eql:type:`relative_duration `,``<>``,``<>``,,,,,,,,,,,,``<>``,,,,,``<>``,,\n:eql:type:`date_duration `,``<>``,``<>``,,,,,,,,,,,,,,,,impl,,,\n:eql:type:`enum`,``<>``,``<>``,,,,,,,,,,,,,,,,,,,\nobject,``<>``,,,,,,,,,,,,,,,,,,,,\n" }, { "path": "docs/reference/reference/edgeql/casts.rst", "content": ".. _ref_eql_casts:\n\n=====\nCasts\n=====\n\nThere are different ways that casts appear in EdgeQL.\n\n\nExplicit Casts\n--------------\n\nA type cast expression converts the specified value to another value of\nthe specified type:\n\n.. eql:synopsis::\n\n \"<\" \">\" \n\nThe :eql:synopsis:`` must be a valid :ref:`type expression\n` denoting a non-abstract scalar or a container type.\n\nFor example, the following expression casts an integer value into a string:\n\n.. code-block:: edgeql-repl\n\n db> select 10;\n {\"10\"}\n\nSee the :eql:op:`type cast operator ` section for more\ninformation on type casting rules.\n\n.. _ref_uuid_casting:\n\n.. lint-off\n\n\nYou can cast a UUID into an object:\n\n.. code-block:: edgeql-repl\n\n db> select '01d9cc22-b776-11ed-8bef-73f84c7e91e7';\n {default::Hero {id: 01d9cc22-b776-11ed-8bef-73f84c7e91e7}}\n\nIf you try to cast a UUID that no object of the type has as its ``id``\nproperty, you'll get an error:\n\n.. code-block:: edgeql-repl\n\n db> select 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa';\n gel error: CardinalityViolationError: 'default::Hero' with id 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa' does not exist\n\n.. lint-on\n\n\nAssignment Casts\n----------------\n\n*Assignment casts* happen when inserting new objects. Numeric types\nwill often be automatically cast into the specific type corresponding\nto the property they are assigned to. This is to avoid extra typing\nwhen dealing with numeric value using fewer bits:\n\n.. code-block:: edgeql\n\n # Automatically cast a literal 42 (which is int64\n # by default) into an int16 value.\n insert MyObject {\n int16_val := 42\n };\n\nIf *assignment* casting is supported for a given pair of types,\n*explicit* casting of those types is also supported.\n\n\nImplicit Casts\n--------------\n\n*Implicit casts* happen automatically whenever the value type doesn't\nmatch the expected type in an expression. This is mostly supported for\nnumeric casts that don't incur any potential information loss (in form\nof truncation), so typically from a less precise type, to a more\nprecise one. The :eql:type:`int64` to :eql:type:`float64` is a notable\nexception, which can suffer from truncation of significant digits for\nvery large integer values. There are a few scenarios when *implicit\ncasts* can occur:\n\n1) Passing arguments that don't match exactly the types in the\n function signature:\n\n .. code-block:: edgeql-repl\n\n db> with x := 12.34\n ... select math::ceil(x);\n {13}\n\n The function :eql:func:`math::ceil` only takes :eql:type:`int64`,\n :eql:type:`float64`, :eql:type:`bigint`, or :eql:type:`decimal` as\n its argument. So the :eql:type:`float32` value will be *implicitly\n cast* into a :eql:type:`float64` in order to match a valid\n signature.\n\n2) Using operands that don't match exactly the types in the\n operator signature (this works the same way as for functions):\n\n .. code-block:: edgeql-repl\n\n db> select 1 + 2.3;\n {3.3}\n\n The operator :eql:op:`+ ` is defined only for operands of\n the same type, so in the expression above the :eql:type:`int64`\n value ``1`` is *implicitly cast* into a :eql:type:`float64` in\n order to match the other operand and produce a valid signature.\n\n3) Mixing different numeric types in a set:\n\n .. code-block:: edgeql-repl\n\n db> select {1, 2.3, 4.5} is float64;\n {true, true, true}\n\n All elements in a set have to be of the same type, so the values\n are cast into :eql:type:`float64` as that happens to be the common\n type to which all the set elements can be *implicitly cast*. This\n would work out the same way if :eql:op:`union` was used instead:\n\n .. code-block:: edgeql-repl\n\n db> select (1 union 2.3 union 4.5) is float64;\n {true, true, true}\n\nIf *implicit* casting is supported for a given pair of types,\n*assignment* and *explicit* casting of those types is also supported.\n\n\n.. _ref_eql_casts_table:\n\nCasting Table\n-------------\n\n.. note::\n\n The UUID-to-object cast is only available since |EdgeDB| 3.0+.\n\n.. This file is automatically generated by `make casts`:\n.. csv-table::\n :file: casts.csv\n :class: vertheadertable\n\n- ``<>`` - can be cast explicitly\n- ``:=`` - assignment cast is supported\n- ``impl`` - implicit cast is supported\n- ``*``- When casting a float type to an integer type, the fractional value\n naturally cannot be preserved after the cast. When executing this cast, we\n round to the nearest integer, rounding ties to the nearest even (e.g., 1.5 is\n rounded up to 2; 2.5 is also rounded to 2).\n" }, { "path": "docs/reference/reference/edgeql/delete.rst", "content": ".. _ref_eql_statements_delete:\n\nDelete\n======\n\n:eql-statement:\n:eql-haswith:\n\n``delete`` -- remove objects from a database.\n\n.. eql:synopsis::\n\n [ with [, ...] ]\n\n delete \n\n [ filter ]\n\n [ order by [direction] [then ...] ]\n\n [ offset ]\n\n [ limit ] ;\n\n:eql:synopsis:`with`\n Alias declarations.\n\n The ``with`` clause allows specifying module aliases as well\n as expression aliases that can be referenced by the ``delete``\n statement. See :ref:`ref_eql_statements_with` for more information.\n\n:eql:synopsis:`delete ...`\n The entire :eql:synopsis:`delete ...` statement is syntactic\n sugar for ``delete (select ...)``. Therefore, the base\n :eql:synopsis:`` and the following :eql:synopsis:`filter`,\n :eql:synopsis:`order by`, :eql:synopsis:`offset`, and\n :eql:synopsis:`limit` clauses shape the set to\n be deleted the same way an explicit :eql:stmt:`select` would.\n\n\nOutput\n~~~~~~\n\nOn successful completion, a ``delete`` statement returns the set\nof deleted objects.\n\n\nExamples\n~~~~~~~~\n\nHere's a simple example of deleting a specific user:\n\n.. code-block:: edgeql\n\n with module example\n delete User\n filter User.name = 'Alice Smith';\n\nAnd here's the equivalent ``delete (select ...)`` statement:\n\n.. code-block:: edgeql\n\n with module example\n delete (select User\n filter User.name = 'Alice Smith');\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`EdgeQL > Delete `\n * - :ref:`Cheatsheets > Deleting data `\n" }, { "path": "docs/reference/reference/edgeql/describe.rst", "content": ".. _ref_eql_statements_describe:\n\nDescribe\n========\n\n:eql-statement:\n\n``describe`` -- provide human-readable description of a schema or a\nschema object\n\n.. eql:synopsis::\n\n describe schema [ as {ddl | sdl | test [ verbose ]} ];\n\n describe [ as {ddl | sdl | text [ verbose ]} ];\n\n # where is one of\n\n object\n annotation\n constraint\n function\n link\n module\n property\n scalar type\n type\n\nDescription\n-----------\n\n``describe`` generates a human-readable description of a schema object.\n\nThe output of a ``describe`` command is a :eql:type:`str` , although\nit cannot be used as an expression in queries.\n\nThere are three output formats to choose from:\n\n:eql:synopsis:`as ddl`\n Provide a valid :ref:`DDL ` definition.\n\n The :ref:`DDL ` generated is a complete valid\n definition of the particular schema object assuming all the other\n referenced schema objects already exist.\n\n This is the default format.\n\n:eql:synopsis:`as sdl`\n Provide an :ref:`SDL ` definition.\n\n The :ref:`SDL ` generated is a complete valid\n definition of the particular schema object assuming all the other\n referenced schema objects already exist.\n\n:eql:synopsis:`as text [verbose]`\n Provide a human-oriented definition.\n\n The human-oriented definition generated is similar to :ref:`SDL\n `, but it includes all the details that are inherited\n (if any).\n\n The :eql:synopsis:`verbose` mode enables displaying additional\n details, such as :ref:`annotations `\n and :ref:`constraints `, which are\n otherwise omitted.\n\nWhen the ``describe`` command is used with the :eql:synopsis:`schema`\nthe result is a definition of the entire database schema. Only the\n:eql:synopsis:`as ddl` option is available for schema description.\n\nThe ``describe`` command can specify the type of schema object that it\nshould generate the description of:\n\n:eql:synopsis:`object `\n Match any module level schema object with the specified *name*.\n\n This is the most general use of the ``describe`` command. It does\n not match :ref:`modules ` (and other\n globals that cannot be uniquely identified just by the name).\n\n:eql:synopsis:`annotation `\n Match only :ref:`annotations ` with the\n specified *name*.\n\n:eql:synopsis:`constraint `\n Match only :ref:`constraints ` with the\n specified *name*.\n\n:eql:synopsis:`function `\n Match only :ref:`functions ` with the\n specified *name*.\n\n:eql:synopsis:`link `\n Match only :ref:`links ` with the specified *name*.\n\n:eql:synopsis:`module `\n Match only :ref:`modules ` with the\n specified *name*.\n\n:eql:synopsis:`property `\n Match only :ref:`properties ` with the\n specified *name*.\n\n:eql:synopsis:`scalar type `\n Match only :ref:`scalar types ` with the\n specified *name*.\n\n:eql:synopsis:`type `\n Match only :ref:`object types ` with the\n specified *name*.\n\n\nExamples\n--------\n\nConsider the following schema:\n\n.. code-block:: sdl\n\n abstract type Named {\n required name: str {\n delegated constraint exclusive;\n }\n }\n\n type User extending Named {\n required email: str {\n annotation title := 'Contact email';\n }\n }\n\nHere are some examples of a ``describe`` command:\n\n.. code-block:: edgeql-repl\n\n db> describe object User;\n {\n \"create type default::User extending default::Named {\n create required single property email -> std::str {\n create annotation std::title := 'Contact email';\n };\n };\"\n }\n db> describe object User as sdl;\n {\n \"type default::User extending default::Named {\n required single property email -> std::str {\n annotation std::title := 'Contact email';\n };\n };\"\n }\n db> describe object User as text;\n {\n 'type default::User extending default::Named {\n required single link __type__ -> schema::Type {\n readonly := true;\n };\n required single property email -> std::str;\n required single property id -> std::uuid {\n readonly := true;\n };\n required single property name -> std::str;\n };'\n }\n db> describe object User as text verbose;\n {\n \"type default::User extending default::Named {\n required single link __type__ -> schema::Type {\n readonly := true;\n };\n required single property email -> std::str {\n annotation std::title := 'Contact email';\n };\n required single property id -> std::uuid {\n readonly := true;\n constraint std::exclusive;\n };\n required single property name -> std::str {\n constraint std::exclusive;\n };\n };\"\n }\n db> describe schema;\n {\n \"create module default if not exists;\n create abstract type default::Named {\n create required single property name -> std::str {\n create delegated constraint std::exclusive;\n };\n };\n create type default::User extending default::Named {\n create required single property email -> std::str {\n create annotation std::title := 'Contact email';\n };\n };\"\n }\n\nThe ``describe`` command also warns you if there are standard library\nmatches that are masked by some user-defined object. Consider the\nfollowing schema:\n\n.. code-block:: sdl\n\n module default {\n function len(v: tuple) -> float64 using (\n select (v.0 ^ 2 + v.1 ^ 2) ^ 0.5\n );\n }\n\nSo within the ``default`` module the user-defined function ``len``\n(computing the length of a vector) masks the built-ins:\n\n.. code-block:: edgeql-repl\n\n db> describe function len as text;\n {\n 'function default::len(v: tuple) ->\n std::float64 using (select\n (((v.0 ^ 2) + (v.1 ^ 2)) ^ 0.5)\n );\n\n # The following builtins are masked by the above:\n\n # function std::len(array: array) -> std::int64 {\n # volatility := \\'Immutable\\';\n # annotation std::description := \\'A polymorphic function to calculate\n a \"length\" of its first argument.\\';\n # using sql $$\n # SELECT cardinality(\"array\")::bigint\n # $$\n # ;};\n # function std::len(bytes: std::bytes) -> std::int64 {\n # volatility := \\'Immutable\\';\n # annotation std::description := \\'A polymorphic function to calculate\n a \"length\" of its first argument.\\';\n # using sql $$\n # SELECT length(\"bytes\")::bigint\n # $$\n # ;};\n # function std::len(str: std::str) -> std::int64 {\n # volatility := \\'Immutable\\';\n # annotation std::description := \\'A polymorphic function to calculate\n a \"length\" of its first argument.\\';\n # using sql $$\n # SELECT char_length(\"str\")::bigint\n # $$\n # ;};',\n }\n" }, { "path": "docs/reference/reference/edgeql/eval.rst", "content": ".. _ref_eql_fundamentals_queries:\n\n====================\nEvaluation algorithm\n====================\n\nEdgeQL is a functional language in the sense that every expression is\na composition of one or more queries.\n\nQueries can be *explicit*, such as a :eql:stmt:`select` statement,\nor *implicit*, as dictated by the semantics of a function, operator or\na statement clause.\n\nAn implicit ``select`` subquery is assumed in the following situations:\n\n- expressions passed as an argument for an aggregate function parameter\n or operand;\n\n- the right side of the assignment operator (``:=``) in expression\n aliases and :ref:`shape element declarations `;\n\n- the majority of statement clauses.\n\nA nested query is called a *subquery*. Here, the phrase\n\"*apearing directly in the query*\" means\n\"appearing directly in the query rather than in the subqueries\".\n\n.. _ref_eql_fundamentals_eval_algo:\n\nA query is evaluated recursively using the following procedure:\n\n1. Make a list of simple paths (i.e., paths that begin with a set reference)\n appearing directly the query. For every path in the list, find all paths\n which begin with the same set reference and treat their longest common\n prefix as an equivalent set reference.\n\n Example:\n\n .. code-block:: edgeql\n\n select (\n User.firstname,\n User.friends.firstname,\n User.friends.lastname,\n Issue.priority.name,\n Issue.number,\n Status.name\n );\n\n In the above query, the longest common prefixes are: ``User``,\n ``User.friends``, ``Issue``, and ``Status.name``.\n\n2. Make a *query input list* of all unique set references which appear\n directly in the query (including the common path prefixes identified above).\n The set references and path prefixes in this list are called *input\n set references*, and the sets they represent are called *input\n sets*. Order this list such that any input references come before\n any other input set reference for which it is a prefix (sorting\n lexicographically works).\n\n3. Compute a set of *input tuples*.\n\n - Begin with a set containing a single empty tuple.\n - For each input set reference, we compute a *dependent* Cartesian\n product of the input tuple set (``X``) so far and the input set\n ``Y`` being considered. In this dependent product, we pair each\n tuple ``x`` in the input tuple set ``X`` with each element of the\n subset of the input set ``Y`` corresponding to the tuple ``x``. (For\n example, in the above example, computing the dependent product\n of User and User.friends would pair each user with all of their\n friends.)\n\n (Mathematically, ``X' = {(x, y) | x āˆˆ X, y āˆˆ f(x)}``, if ``f(x)``\n selects the appropriate subset.)\n\n The set produced becomes the new input tuple set and we continue\n down the list.\n - As a caveat to the above, if an input set appears exclusively as\n an :ref:`optional ` argument, it produces\n pairs with a placeholder value ``Missing`` instead of an empty\n Cartesian product in the above\n set. (Mathematically, this corresponds to having ``f(x) =\n {Missing}`` whenever it would otherwise produce an empty set.)\n\n4. Iterate over the set of input tuples, and on every iteration:\n\n - in the query and its subqueries, replace each input set reference with the\n corresponding value from the input tuple or an empty set if the value\n is ``Missing``;\n\n - evaluate the query expression in the order of precedence using\n the following rules:\n\n * subqueries are evaluated recursively from step 1;\n\n * a function or an operator is evaluated in a loop over a Cartesian\n product of its non-aggregate arguments\n (empty ``optional`` arguments are excluded from the product);\n aggregate arguments are passed as a whole set;\n the results of the invocations are collected to form a single set.\n\n5. Collect the results of all iterations to obtain the final result set.\n" }, { "path": "docs/reference/reference/edgeql/for.rst", "content": ".. _ref_eql_statements_for:\n\nFor\n===\n\n:eql-statement:\n:eql-haswith:\n\n\n``for``--compute a union of subsets based on values of another set\n\n.. eql:synopsis::\n\n [ with [, ...] ]\n\n for in \n\n union ;\n\n:eql:synopsis:`for in `\n The ``for`` clause has this general form:\n\n .. TODO: rewrite this\n\n .. eql:synopsis::\n\n for in \n\n where :eql:synopsis:`` is a\n\t:ref:`literal `,\n\ta :ref:`function call `,\n\ta :ref:`set constructor `,\n\ta :ref:`path `,\n\tor any parenthesized expression or statement.\n\n:eql:synopsis:`union `\n The ``union`` clause of the ``for`` statement has this general form:\n\n .. TODO: rewrite this\n\n .. eql:synopsis::\n\n union \n\n Here, :eql:synopsis:``\n is an arbitrary expression that is evaluated for\n every element in a set produced by evaluating the ``for`` clause.\n The results of the evaluation are appended to the result set.\n\n\n.. _ref_eql_forstatement:\n\nUsage of ``for`` statement\n++++++++++++++++++++++++++\n\n``for`` statement has some powerful features that deserve to be\nconsidered in detail separately. However, the common core is that\n``for`` iterates over elements of some arbitrary expression. Then for\neach element of the iterator some set is computed and combined via a\n:eql:op:`union` with the other such computed sets.\n\nThe simplest use case is when the iterator is given by a set\nexpression and it follows the general form of ``for x in A ...``:\n\n.. code-block:: edgeql\n\n with module example\n # the iterator is an explicit set of tuples, so x is an\n # element of this set, i.e. a single tuple\n for x in {\n (name := 'Alice', theme := 'fire'),\n (name := 'Bob', theme := 'rain'),\n (name := 'Carol', theme := 'clouds'),\n (name := 'Dave', theme := 'forest')\n }\n # typically this is used with an INSERT, DELETE or UPDATE\n union (\n insert\n User {\n name := x.name,\n theme := x.theme,\n }\n );\n\nSince ``x`` is an element of a set it is guaranteed to be a non-empty\nsingleton in all of the expressions used by the ``union`` and later\nclauses of ``for``.\n\nAnother variation this usage of ``for`` is a bulk ``update``. There\nare cases when a bulk update involves a lot of external data that\ncannot be derived from the objects being updated. That is a good\nuse-case when a ``for`` statement is appropriate.\n\n.. code-block:: edgeql\n\n # Here's an example of an update that is awkward to\n # express without the use of FOR statement\n with module example\n update User\n filter .name in {'Alice', 'Bob', 'Carol', 'Dave'}\n set {\n theme := 'red' if .name = 'Alice' else\n 'star' if .name = 'Bob' else\n 'dark' if .name = 'Carol' else\n 'strawberry'\n };\n\n # Using a FOR statement, the above update becomes simpler to\n # express or review for a human.\n with module example\n for x in {\n (name := 'Alice', theme := 'red'),\n (name := 'Bob', theme := 'star'),\n (name := 'Carol', theme := 'dark'),\n (name := 'Dave', theme := 'strawberry')\n }\n union (\n update User\n filter .name = x.name\n set {\n theme := x.theme\n }\n );\n\nWhen updating data that mostly or completely depends on the objects\nbeing updated there's no need to use the ``for`` statement and it is not\nadvised to use it for performance reasons.\n\n.. code-block:: edgeql\n\n with module example\n update User\n filter .name in {'Alice', 'Bob', 'Carol', 'Dave'}\n set {\n theme := 'halloween'\n };\n\n # The above can be accomplished with a for statement,\n # but it is not recommended.\n with module example\n for x in {'Alice', 'Bob', 'Carol', 'Dave'}\n union (\n update User\n filter .name = x\n set {\n theme := 'halloween'\n }\n );\n\nAnother example of using a ``for`` statement is working with link\nproperties. Specifying the link properties either at creation time or\nin a later step with an update is often simpler with a ``for``\nstatement helping to associate the link target to the link property in\nan intuitive manner.\n\n.. code-block:: edgeql\n\n # Expressing this without for statement is fairly tedious.\n with\n module example,\n U2 := User\n for x in {\n (\n name := 'Alice',\n friends := [('Bob', 'coffee buff'),\n ('Carol', 'dog person')]\n ),\n (\n name := 'Bob',\n friends := [('Alice', 'movie buff'),\n ('Dave', 'cat person')]\n )\n }\n union (\n update User\n filter .name = x.name\n set {\n friends := assert_distinct(\n (\n for f in array_unpack(x.friends)\n union (\n select U2 {@nickname := f.1}\n filter U2.name = f.0\n )\n )\n )\n }\n );\n\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`EdgeQL > For `\n" }, { "path": "docs/reference/reference/edgeql/functions.rst", "content": ".. _ref_reference_function_call:\n\n\nFunction calls\n==============\n\n\n|Gel| provides a number of functions in the :ref:`standard library\n`. It is also possible for users to :ref:`define their own\n` functions.\n\n\nThe syntax for a function call is as follows:\n\n.. eql:synopsis::\n\n \"(\" [ [, , ...]] \")\"\n\n # where is:\n\n | := \n\n\n\nHere :eql:synopsis:`` is a possibly qualified name of a\nfunction, and :eql:synopsis:`` is an *expression* optionally\nprefixed with an argument name and the assignment operator (``:=``)\nfor :ref:`named only ` arguments.\n\nFor example, the following computes the length of a string ``'foo'``:\n\n.. code-block:: edgeql-repl\n\n db> select len('foo');\n {3}\n\nAnd here's an example of using a *named only* argument to provide a\ndefault value:\n\n.. code-block:: edgeql-repl\n\n db> select array_get(['hello', 'world'], 10, default := 'n/a');\n {'n/a'}\n\n\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`Schema > Functions `\n * - :ref:`SDL > Functions `\n * - :ref:`DDL > Functions `\n * - :ref:`Introspection > Functions `\n * - :ref:`Cheatsheets > Functions `\n\n" }, { "path": "docs/reference/reference/edgeql/group.rst", "content": ".. _ref_eql_statements_group:\n\nGroup\n=====\n\n:eql-statement:\n:eql-haswith:\n\n``group``--partition a set into subsets based on one or more keys\n\n.. eql:synopsis::\n\n [ with [, ...] ]\n\n group [ := ] \n\n [ using := , [, ...] ]\n\n by , ... ;\n\n # where a is one of\n\n \n { , ... }\n ROLLUP( , ... )\n CUBE( , ... )\n\n # where a is one of\n\n ()\n \n ( , ... )\n\n # where a is one of\n\n \n .\n\n:eql:synopsis:`group `\n The ``group`` clause sets up the input set that will be operated on.\n\n Much like in :eql:stmt:`select` it is possible to define an ad-hoc alias\n at this stage to make referring to the starting set concisely.\n\n:eql:synopsis:`using := `\n The ``using`` clause defines one or more aliases which can then be used as\n part of the grouping key.\n\n If the :eql:synopsis:`by` clause only refers to\n :eql:synopsis:`.` the ``using`` clause is optional.\n\n:eql:synopsis:`by `\n The ``by`` clause sepecifies which parameters will be used to partition\n the starting set.\n\n There are only two basic components for defining\n :eql:synopsis:``: references to\n :eql:synopsis:`` defined in the :eql:synopsis:`using` clause\n or by references to the short-path format of\n :eql:synopsis:`.`. The :eql:synopsis:`.` has to\n refer to properties or links immediately present on the type of starting\n set.\n\n The basic building blocks can also be combined by using parentheses ``(\n )`` to indicate that partitioning will happen based on several parameters\n at once.\n\n It is also possible to specify *grouping sets*, which are denoted using\n curly braces ``{ }``. The results will contain different partitioning\n based on each of the grouping set elements. When there are multiple\n top-level grouping-elements then the cartesian product of them is taken to\n determine the grouping set. Thus ``a, {b, c}`` is equivalent to ``{(a, b),\n (a, c)}`` grouping sets.\n\n :eql:synopsis:`ROLLUP` and :eql:synopsis:`CUBE` are a shorthand to specify\n particular grouping sets. :eql:synopsis:`ROLLUP` groups by all prefixes\n of a list of elements, so ``ROLLUP (a, b, c)`` is equivalent to ``{(),\n (a), (a, b), (a, b, c)}``. :eql:synopsis:`CUBE` groups by all elements of\n the power set, so ``CUBE (a, b)`` is equivalent to ``{(), (a), (b), (a,\n b)}``.\n\n\nOutput\n------\n\nThe ``group`` statement partitions a starting set into subsets based on some\nspecified parameters. The output is organized into a set of :ref:`free objects\n` of the following structure:\n\n.. eql:synopsis::\n\n {\n \"key\": { := [, ...] },\n \"grouping\": ,\n \"elements\": ,\n }\n\n:eql:synopsis:`\"key\"`\n The :eql:synopsis:`\"key\"` contains another :ref:`free object\n `, which contains all the aliases or field\n names used as the key together with the specific values these parameters\n take for this particular subset.\n\n:eql:synopsis:`\"grouping\"`\n The :eql:synopsis:`\"grouping\"` contains a :eql:type:`str` set of all the\n names of the parameters used as the key for this particular subset. This\n is especially useful when using grouping sets and the parameters used in\n the key are not the same for all partitionings.\n\n:eql:synopsis:`\"elements\"`\n The :eql:synopsis:`\"elements\"` contains the actual subset of values that\n match the :eql:synopsis:`\"key\"`.\n\n\nExamples\n--------\n\nHere's a simple example without using any aggregation or any further\nprocessing:\n\n.. code-block:: edgeql-repl\n\n db> group Movie {title} by .release_year;\n {\n {\n key: {release_year: 2016},\n grouping: {'release_year'},\n elements: {\n default::Movie {title: 'Captain America: Civil War'},\n default::Movie {title: 'Doctor Strange'},\n },\n },\n {\n key: {release_year: 2017},\n grouping: {'release_year'},\n elements: {\n default::Movie {title: 'Spider-Man: Homecoming'},\n default::Movie {title: 'Thor: Ragnarok'},\n },\n },\n {\n key: {release_year: 2018},\n grouping: {'release_year'},\n elements: {default::Movie {title: 'Ant-Man and the Wasp'}},\n },\n {\n key: {release_year: 2019},\n grouping: {'release_year'},\n elements: {default::Movie {title: 'Spider-Man: No Way Home'}},\n },\n {\n key: {release_year: 2021},\n grouping: {'release_year'},\n elements: {default::Movie {title: 'Black Widow'}},\n },\n ...\n }\n\nOr we can group by an expression instead, such as whether the title starts\nwith a vowel or not:\n\n.. code-block:: edgeql-repl\n\n db> with\n ... # Apply the group query only to more recent movies\n ... M := (select Movie filter .release_year > 2015)\n ... group M {title}\n ... using vowel := re_test('(?i)^[aeiou]', .title)\n ... by vowel;\n {\n {\n key: {vowel: false},\n grouping: {'vowel'},\n elements: {\n default::Movie {title: 'Thor: Ragnarok'},\n default::Movie {title: 'Doctor Strange'},\n default::Movie {title: 'Spider-Man: Homecoming'},\n default::Movie {title: 'Captain America: Civil War'},\n default::Movie {title: 'Black Widow'},\n default::Movie {title: 'Spider-Man: No Way Home'},\n },\n },\n {\n key: {vowel: true},\n grouping: {'vowel'},\n elements: {default::Movie {title: 'Ant-Man and the Wasp'}},\n },\n }\n\nIt is also possible to group scalars instead of objects, in which case you\nneed to define an ad-hoc alias to refer to the scalar set in order to specify\nhow it will be grouped:\n\n.. code-block:: edgeql-repl\n\n db> with\n ... # Apply the group query only to more recent movies\n ... M := (select Movie filter .release_year > 2015)\n ... group T := M.title\n ... using vowel := re_test('(?i)^[aeiou]', T)\n ... by vowel;\n {\n {\n key: {vowel: false},\n grouping: {'vowel'},\n elements: {\n 'Captain America: Civil War',\n 'Doctor Strange',\n 'Spider-Man: Homecoming',\n 'Thor: Ragnarok',\n 'Spider-Man: No Way Home',\n 'Black Widow',\n },\n },\n {\n key: {vowel: true},\n grouping: {'vowel'},\n elements: {'Ant-Man and the Wasp'}\n },\n }\n\nOften the results of ``group`` are immediately used in a :eql:stmt:`select`\nstatement to provide some kind of analytical results:\n\n.. code-block:: edgeql-repl\n\n db> with\n ... # Apply the group query only to more recent movies\n ... M := (select Movie filter .release_year > 2015),\n ... groups := (\n ... group M {title}\n ... using vowel := re_test('(?i)^[aeiou]', .title)\n ... by vowel\n ... )\n ... select groups {\n ... starts_with_vowel := .key.vowel,\n ... count := count(.elements),\n ... mean_title_length :=\n ... round(math::mean(len(.elements.title)))\n ... };\n {\n {starts_with_vowel: false, count: 6, mean_title_length: 18},\n {starts_with_vowel: true, count: 1, mean_title_length: 20},\n }\n\nIt's possible to group by more than one parameter. For example, we can add the\nrelease decade to whether the ``title`` starts with a vowel:\n\n.. code-block:: edgeql-repl\n\n db> with\n ... # Apply the group query only to more recent movies\n ... M := (select Movie filter .release_year > 2015),\n ... groups := (\n ... group M {title}\n ... using\n ... vowel := re_test('(?i)^[aeiou]', .title),\n ... decade := .release_year // 10\n ... by vowel, decade\n ... )\n ... select groups {\n ... key := .key {vowel, decade},\n ... count := count(.elements),\n ... mean_title_length :=\n ... math::mean(len(.elements.title))\n ... };\n {\n {\n key: {vowel: false, decade: 201},\n count: 5,\n mean_title_length: 19.8,\n },\n {\n key: {vowel: false, decade: 202},\n count: 1,\n mean_title_length: 11,\n },\n {\n key: {vowel: true, decade: 201},\n count: 1,\n mean_title_length: 20\n },\n }\n\nHaving more than one grouping parameter opens up the possibility to using\n*grouping sets* to see the way grouping parameters interact with the analytics\nwe're gathering:\n\n.. code-block:: edgeql-repl\n\n db> with\n ... # Apply the group query only to more recent movies\n ... M := (select Movie filter .release_year > 2015),\n ... groups := (\n ... group M {title}\n ... using\n ... vowel := re_test('(?i)^[aeiou]', .title),\n ... decade := .release_year // 10\n ... by CUBE(vowel, decade)\n ... )\n ... select groups {\n ... key := .key {vowel, decade},\n ... grouping,\n ... count := count(.elements),\n ... mean_title_length :=\n ... (math::mean(len(.elements.title)))\n ... } order by array_agg(.grouping);\n {\n {\n key: {vowel: {}, decade: {}},\n grouping: {},\n count: 7,\n mean_title_length: 18.571428571428573,\n },\n {\n key: {vowel: {}, decade: 202},\n grouping: {'decade'},\n count: 1,\n mean_title_length: 11,\n },\n {\n key: {vowel: {}, decade: 201},\n grouping: {'decade'},\n count: 6,\n mean_title_length: 19.833333333333332,\n },\n {\n key: {vowel: true, decade: {}},\n grouping: {'vowel'},\n count: 1,\n mean_title_length: 20,\n },\n {\n key: {vowel: false, decade: {}},\n grouping: {'vowel'},\n count: 6,\n mean_title_length: 18.333333333333332,\n },\n {\n key: {vowel: false, decade: 201},\n grouping: {'vowel', 'decade'},\n count: 5,\n mean_title_length: 19.8,\n },\n {\n key: {vowel: true, decade: 201},\n grouping: {'vowel', 'decade'},\n count: 1,\n mean_title_length: 20,\n },\n {\n key: {vowel: false, decade: 202},\n grouping: {'vowel', 'decade'},\n count: 1,\n mean_title_length: 11,\n },\n }\n\n\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`EdgeQL > Group `\n" }, { "path": "docs/reference/reference/edgeql/index.rst", "content": ".. _ref_eql_statements:\n\nEdgeQL\n========\n\nStatements in EdgeQL are a kind of an *expression* that has one or\nmore ``clauses`` and is used to retrieve or modify data in a database.\n\nQuery statements:\n\n* :eql:stmt:`select`\n\n Retrieve data from a database and compute arbitrary expressions.\n\n* :eql:stmt:`for`\n\n Compute an expression for every element of an input set and\n concatenate the results.\n\n* :eql:stmt:`group`\n\n Group data into subsets by keys.\n\nData modification statements:\n\n* :eql:stmt:`insert`\n\n Create new object in a database.\n\n* :eql:stmt:`update`\n\n Update objects in a database.\n\n* :eql:stmt:`delete`\n\n Remove objects from a database.\n\nTransaction control statements:\n\n* :eql:stmt:`start transaction`\n\n Start a transaction.\n\n* :eql:stmt:`commit`\n\n Commit the current transaction.\n\n* :eql:stmt:`rollback`\n\n Abort the current transaction.\n\n* :eql:stmt:`declare savepoint`\n\n Declare a savepoint within the current transaction.\n\n* :eql:stmt:`rollback to savepoint`\n\n Rollback to a savepoint within the current transaction.\n\n* :eql:stmt:`release savepoint`\n\n Release a previously declared savepoint.\n\nSession state control statements:\n\n* :eql:stmt:`set` and :eql:stmt:`reset`.\n\nIntrospection command:\n\n* :eql:stmt:`describe`.\n\nPerformance analysis statement:\n\n* :eql:stmt:`analyze`.\n\n\n.. toctree::\n :maxdepth: 3\n :hidden:\n\n\n lexical\n eval\n shapes\n paths\n casts\n functions\n cardinality\n volatility\n\n select\n insert\n update\n delete\n for\n group\n with\n analyze\n\n tx_start\n tx_commit\n tx_rollback\n tx_sp_declare\n tx_sp_release\n tx_sp_rollback\n\n sess_set_alias\n sess_reset_alias\n\n describe\n" }, { "path": "docs/reference/reference/edgeql/insert.rst", "content": ".. _ref_eql_statements_insert:\n\nInsert\n======\n\n:eql-statement:\n:eql-haswith:\n\n``insert`` -- create a new object in a database\n\n.. eql:synopsis::\n\n [ with [ , ... ] ]\n insert [ ]\n [ unless conflict\n [ on [ else ] ]\n ] ;\n\n\nDescription\n-----------\n\n``insert`` inserts a new object into a database.\n\nWhen evaluating an ``insert`` statement, *expression* is used solely to\ndetermine the *type* of the inserted object and is not evaluated in any\nother way.\n\nIf a value for a *required* link is evaluated to an empty set, an error is\nraised.\n\nIt is possible to insert multiple objects by putting the ``insert``\ninto a :eql:stmt:`for` statement.\n\nSee :ref:`ref_eql_forstatement` for more details.\n\n:eql:synopsis:`with`\n Alias declarations.\n\n The ``with`` clause allows specifying module aliases as well\n as expression aliases that can be referenced by the :eql:stmt:`update`\n statement. See :ref:`ref_eql_statements_with` for more information.\n\n:eql:synopsis:``\n An arbitrary expression returning a set of objects to be updated.\n\n .. eql:synopsis::\n\n insert \n [ \"{\" := [, ...] \"}\" ]\n\n.. _ref_eql_statements_conflict:\n\n:eql:synopsis:`unless conflict [ on ]`\n :index: unless conflict\n\n Handler of conflicts.\n\n This clause allows to handle specific conflicts arising during\n execution of ``insert`` without producing an error. If the\n conflict arises due to exclusive constraints on the properties\n specified by *property-expr*, then instead of failing with an\n error the ``insert`` statement produces an empty set (or an\n alternative result).\n\n The exclusive constraint on ```` cannot be defined on a\n parent type.\n\n The specified *property-expr* may be either a reference to a property (or\n link) or a tuple of references to properties (or links).\n\n A caveat, however, is that ``unless conflict`` will not prevent\n conflicts caused between multiple DML operations in the same\n query; inserting two conflicting objects (through use of ``for``\n or simply with two ``insert`` statements) will cause a constraint\n error.\n\n Example:\n\n .. code-block:: edgeql\n\n insert User { email := 'user@example.org' }\n unless conflict on .email\n\n .. code-block:: edgeql\n\n insert User { first := 'Jason', last := 'Momoa' }\n unless conflict on (.first, .last)\n\n:eql:synopsis:`else `\n Alternative result in case of conflict.\n\n This clause can only appear after ``unless conflict`` clause. Any\n valid expression can be specified as the *alternative*. When a\n conflict arises, the result of the ``insert`` becomes the\n *alternative* expression (instead of the default ``{}``).\n\n In order to refer to the conflicting object in the *alternative*\n expression, the name used in the ``insert`` must be used (see\n :ref:`example below `).\n\nOutputs\n-------\n\nThe result of an ``insert`` statement used as an *expression* is a\nsingleton set containing the inserted object.\n\n\nExamples\n--------\n\nHere's a simple example of an ``insert`` statement creating a new user:\n\n.. code-block:: edgeql\n\n with module example\n insert User {\n name := 'Bob Johnson'\n };\n\n``insert`` is not only a statement, but also an expression and as such\nis has a value of the set of objects that has been created.\n\n.. code-block:: edgeql\n\n with module example\n insert Issue {\n number := '100',\n body := 'Fix errors in insert',\n owner := (\n select User filter User.name = 'Bob Johnson'\n )\n };\n\nIt is possible to create nested objects in a single ``insert``\nstatement as an atomic operation.\n\n.. code-block:: edgeql\n\n with module example\n insert Issue {\n number := '101',\n body := 'Nested insert',\n owner := (\n insert User {\n name := 'Nested User'\n }\n )\n };\n\nThe above statement will create a new ``Issue`` as well as a new\n``User`` as the owner of the ``Issue``. It will also return the new\n``Issue`` linked to the new ``User`` if the statement is used as an\nexpression.\n\nIt is also possible to create new objects based on some existing data\neither provided as an explicit list (possibly automatically generated\nby some tool) or a query. A ``for`` statement is the basis for this\nuse-case and ``insert`` is simply the expression in the ``union``\nclause.\n\n.. code-block:: edgeql\n\n # example of a bulk insert of users based on explicitly provided\n # data\n with module example\n for x in {'Alice', 'Bob', 'Carol', 'Dave'}\n union (insert User {\n name := x\n });\n\n\n # example of a bulk insert of issues based on a query\n with\n module example,\n Elvis := (select User filter .name = 'Elvis'),\n Open := (select Status filter .name = 'Open')\n\n for Q in (select User filter .name ilike 'A%')\n\n union (insert Issue {\n name := Q.name + ' access problem',\n body := 'This user was affected by recent system glitch',\n owner := Elvis,\n status := Open\n });\n\n.. _ref_eql_statements_insert_unless:\n\nThere's an important use-case where it is necessary to either insert a\nnew object or update an existing one identified with some key. This is\nwhat the ``unless conflict`` clause allows:\n\n.. code-block:: edgeql\n\n with module people\n select (\n insert Person {\n name := \"Łukasz Langa\", is_admin := true\n }\n unless conflict on .name\n else (\n update Person\n set { is_admin := true }\n )\n ) {\n name,\n is_admin\n };\n\n\n.. note::\n\n Statements in EdgeQL represent an atomic interaction with the\n database. From the point of view of a statement all side-effects\n (such as database updates) happen after the statement is executed.\n So as far as each statement is concerned, it is some purely\n functional expression evaluated on some specific input (database\n state).\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`EdgeQL > Insert `\n * - :ref:`Cheatsheets > Inserting data `\n" }, { "path": "docs/reference/reference/edgeql/lexical.rst", "content": ".. _ref_eql_lexical:\n\n\nLexical structure\n=================\n\nEvery EdgeQL command is composed of a sequence of *tokens*, terminated by\na semicolon (``;``). The types of valid tokens as well as their order\nis determined by the syntax of the particular command.\n\nEdgeQL is case sensistive except for *keywords* (in the examples the\nkeywords are written in upper case as a matter of convention).\n\nThere are several kinds of tokens: *keywords*, *identifiers*,\n*literals* (constants) and *symbols* (operators and punctuation).\n\nTokens are normally separated by whitespace (space, tab, newline) or\ncomments.\n\n\nIdentifiers\n-----------\n\nThere are two ways of writing identifiers in EdgeQL: plain and quoted.\nThe plain identifiers are similar to many other languages, they are\nalphanumeric with underscores and cannot start with a digit. The\nquoted identifiers start and end with a *backtick*\n```quoted.identifier``` and can contain any characters inside with a\nfew exceptions. They must not start with an ampersand (``@``) or\ncontain a double colon (``::``). If there's a need to include a backtick\ncharacter as part of the identifier name a double-backtick sequence\n(``````) should be used: ```quoted``identifier``` will result in the\nactual identifier being ``quoted`identifier``.\n\n.. productionlist:: edgeql\n identifier: `plain_ident` | `quoted_ident`\n plain_ident: `ident_first` `ident_rest`*\n ident_first: \n ident_rest: \n quoted_ident: \"`\" `qident_first` `qident_rest`* \"`\"\n qident_first: \n qident_rest: \n\nQuoted identifiers are usually needed to represent module names that\ncontain a dot (``.``) or to distinguish *names* from *reserved keywords*\n(for instance to allow referring to a link named \"order\" as ```order```).\n\n\n.. _ref_eql_lexical_names:\n\nNames and keywords\n------------------\n\n.. TODO::\n\n This section needs a significant update.\n\nThere are a number of *reserved* and *unreserved* keywords in EdgeQL.\nEvery identifier that is not a *reserved* keyword is a valid *name*.\n*Names* are used to refer to concepts, links, link properties, etc.\n\n.. TODO: update this for \"branch\"\n\n.. productionlist:: edgeql\n short_name: `not_keyword_ident` | `quoted_ident`\n not_keyword_ident: \n keyword: `reserved_keyword` | `unreserved_keyword`\n reserved_keyword: case insensitive sequence matching any\n : of the following\n : \"AGGREGATE\" | \"ALTER\" | \"AND\" |\n : \"ANY\" | \"COMMIT\" | \"CREATE\" |\n : \"DELETE\" | \"DETACHED\" | \"DISTINCT\" |\n : \"DROP\" | \"ELSE\" | \"EMPTY\" | \"EXISTS\" |\n : \"FALSE\" | \"FILTER\" | \"FUNCTION\" |\n : \"GET\" | \"GROUP\" | \"IF\" | \"ILIKE\" |\n : \"IN\" | \"INSERT\" | \"IS\" | \"LIKE\" |\n : \"LIMIT\" | \"MODULE\" | \"NOT\" | \"OFFSET\" |\n : \"OR\" | \"ORDER\" | \"OVER\" |\n : \"PARTITION\" | \"ROLLBACK\" | \"SELECT\" |\n : \"SET\" | \"SINGLETON\" | \"START\" | \"TRUE\" |\n : \"UPDATE\" | \"UNION\" | \"WITH\"\n unreserved_keyword: case insensitive sequence matching any\n : of the following\n : \"ABSTRACT\" | \"ACTION\" | \"AFTER\" |\n : \"ARRAY\" | \"AS\" | \"ASC\" | \"ATOM\" |\n : \"ANNOTATION\" | \"BEFORE\" | \"BY\" |\n : \"CONCEPT\" | \"CONSTRAINT\" |\n : \"DATABASE\" | \"DESC\" | \"EVENT\" |\n : \"EXTENDING\" | \"FINAL\" | \"FIRST\" |\n : \"FOR\" | \"FROM\" | \"INDEX\" |\n : \"INITIAL\" | \"LAST\" | \"LINK\" |\n : \"MAP\" | \"MIGRATION\" | \"OF\" | \"ON\" |\n : \"POLICY\" | \"PROPERTY\" |\n : \"REQUIRED\" | \"RENAME\" | \"TARGET\" |\n : \"THEN\" | \"TO\" | \"TRANSACTION\" |\n : \"TUPLE\" | \"VALUE\" | \"VIEW\"\n\nFully-qualified names consist of a module, ``::``, and a short name.\nThey can be used in most places where a short name can appear (such as\npaths and shapes).\n\n.. productionlist:: edgeql\n name: `short_name` | `fq_name`\n fq_name: `short_name` \"::\" `short_name` |\n : `short_name` \"::\" `unreserved_keyword`\n\n\n.. _ref_eql_lexical_const:\n\nConstants\n---------\n\nA number of scalar types have literal constant expressions.\n\n\n.. _ref_eql_lexical_str:\n\nStrings\n^^^^^^^\n\nProduction rules for :eql:type:`str` literals:\n\n.. productionlist:: edgeql\n string: `str` | `raw_str`\n str: \"'\" `str_content`* \"'\" | '\"' `str_content`* '\"'\n raw_str: \"r'\" `raw_content`* \"'\" |\n : 'r\"' `raw_content`* '\"' |\n : `dollar_quote` `raw_content`* `dollar_quote`\n raw_content: \n dollar_quote: \"$\" `q_char0`? `q_char`* \"$\"\n q_char0: \"A\"...\"Z\" | \"a\"...\"z\" | \"_\"\n q_char: \"A\"...\"Z\" | \"a\"...\"z\" | \"_\" | \"0\"...\"9\"\n str_content: | `unicode` | `str_escapes`\n unicode: \n str_escapes: \n\nThe inclusion of \"high ASCII\" character in :token:`edgeql:q_char` in\npractice reflects the ability to use some of the letters with\ndiacritics like ``Ć²`` or ``Ć¼`` in the dollar-quote delimiter.\n\nHere's a list of valid :token:`edgeql:str_escapes`:\n\n.. _ref_eql_lexical_str_escapes:\n\n+--------------------+---------------------------------------------+\n| Escape Sequence | Meaning |\n+====================+=============================================+\n| ``\\[newline]`` | Backslash and all whitespace up to next |\n| | non-whitespace character is ignored |\n+--------------------+---------------------------------------------+\n| ``\\\\`` | Backslash (\\\\) |\n+--------------------+---------------------------------------------+\n| ``\\'`` | Single quote (') |\n+--------------------+---------------------------------------------+\n| ``\\\"`` | Double quote (\") |\n+--------------------+---------------------------------------------+\n| ``\\b`` | ASCII backspace (``\\x08``) |\n+--------------------+---------------------------------------------+\n| ``\\f`` | ASCII form feed (``\\x0C``) |\n+--------------------+---------------------------------------------+\n| ``\\n`` | ASCII newline (``\\x0A``) |\n+--------------------+---------------------------------------------+\n| ``\\r`` | ASCII carriage return (``\\x0D``) |\n+--------------------+---------------------------------------------+\n| ``\\t`` | ASCII tabulation (``\\x09``) |\n+--------------------+---------------------------------------------+\n| ``\\xhh`` | Character with hex value hh |\n+--------------------+---------------------------------------------+\n| ``\\uhhhh`` | Character with 16-bit hex value hhhh |\n+--------------------+---------------------------------------------+\n| ``\\Uhhhhhhhh`` | Character with 32-bit hex value hhhhhhhh |\n+--------------------+---------------------------------------------+\n\nHere's some examples of regular strings using escape sequences\n\n.. code-block:: edgeql-repl\n\n db> select 'hello\n ... world';\n {'hello\n world'}\n\n db> select \"hello\\nworld\";\n {'hello\n world'}\n\n db> select 'hello \\\n ... world';\n {'hello world'}\n\n db> select 'https://geldata.com/\\\n ... docs/edgeql/lexical\\\n ... #constants';\n {'https://geldata.com/docs/edgeql/lexical#constants'}\n\n db> select 'hello \\\\ world';\n {'hello \\ world'}\n\n db> select 'hello \\'world\\'';\n {\"hello 'world'\"}\n\n db> select 'hello \\x77orld';\n {'hello world'}\n\n db> select 'hello \\u0077orld';\n {'hello world'}\n\n.. _ref_eql_lexical_raw:\n\nRaw strings don't have any specially interpreted symbols; they contain\nall the symbols between the quotes exactly as typed.\n\n.. code-block:: edgeql-repl\n\n db> select r'hello \\\\ world';\n {'hello \\\\ world'}\n\n db> select r'hello \\\n ... world';\n {'hello \\\n world'}\n\n db> select r'hello\n ... world';\n {'hello\n world'}\n\n.. _ref_eql_lexical_dollar_quoting:\n\nDollar-quoted String Constants\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nA special case of raw strings are *dollar-quoted* strings. They allow\nusing either kind of quote symbols ``'`` or ``\"`` as part of the\nstring content without the quotes terminating the string. In fact,\nbecause the *dollar-quote* delimiter sequences can have arbitrary\nalphanumeric additional fillers, it is always possible to surround any\ncontent with *dollar-quotes* in an unambiguous manner:\n\n.. code-block:: edgeql-repl\n\n db> select $$hello\n ... world$$;\n {'hello\n world'}\n\n db> select $$hello\\nworld$$;\n {'hello\\nworld'}\n\n db> select $$\"hello\" 'world'$$;\n {\"\\\"hello\\\" 'world'\"}\n\n db> select $a$hello$$world$$$a$;\n {'hello$$world$$'}\n\nMore specifically, a delimiter:\n\n* Must start with an ASCII letter or underscore\n* Has following characters that can be digits 0-9, underscores or\n ASCII letters\n\n.. _ref_eql_lexical_bytes:\n\nBytes\n^^^^^\n\nProduction rules for :eql:type:`bytes` literals:\n\n.. productionlist:: edgeql\n bytes: \"b'\" `bytes_content`* \"'\" | 'b\"' `bytes_content`* '\"'\n bytes_content: | `ascii` | `bytes_escapes`\n ascii: \n bytes_escapes: \n\nHere's a list of valid :token:`edgeql:bytes_escapes`:\n\n.. _ref_eql_lexical_bytes_escapes:\n\n+--------------------+---------------------------------------------+\n| Escape Sequence | Meaning |\n+====================+=============================================+\n| ``\\\\`` | Backslash (\\\\) |\n+--------------------+---------------------------------------------+\n| ``\\'`` | Single quote (') |\n+--------------------+---------------------------------------------+\n| ``\\\"`` | Double quote (\") |\n+--------------------+---------------------------------------------+\n| ``\\b`` | ASCII backspace (``\\x08``) |\n+--------------------+---------------------------------------------+\n| ``\\f`` | ASCII form feed (``\\x0C``) |\n+--------------------+---------------------------------------------+\n| ``\\n`` | ASCII newline (``\\x0A``) |\n+--------------------+---------------------------------------------+\n| ``\\r`` | ASCII carriage return (``\\x0D``) |\n+--------------------+---------------------------------------------+\n| ``\\t`` | ASCII tabulation (``\\x09``) |\n+--------------------+---------------------------------------------+\n| ``\\xhh`` | Character with hex value hh |\n+--------------------+---------------------------------------------+\n\n\nIntegers\n^^^^^^^^\n\nThere are two kinds of integer constants: limited size\n(:eql:type:`int64`) and unlimited size (:eql:type:`bigint`). Unlimited\nsize integer :eql:type:`bigint` literals are similar to a regular\ninteger literals with an ``n`` suffix. The production rules are as\nfollows:\n\n.. productionlist:: edgeql\n bigint: `integer` \"n\"\n integer: \"0\" | `non_zero` `digit`*\n non_zero: \"1\"...\"9\"\n digit: \"0\"...\"9\"\n\nBy default all integer literals are interpreted as :eql:type:`int64`,\nwhile an explicit cast can be used to convert them to :eql:type:`int16`\nor :eql:type:`int32`:\n\n.. code-block:: edgeql-repl\n\n db> select 0;\n {0}\n\n db> select 123;\n {123}\n\n db> select 456;\n {456}\n\n db> select 789;\n {789}\n\nExamples of :eql:type:`bigint` literals:\n\n.. code-block:: edgeql-repl\n\n db> select 123n;\n {123n}\n\n db> select 12345678901234567890n;\n {12345678901234567890n}\n\n\nReal Numbers\n^^^^^^^^^^^^\n\nJust as for integers, there are two kinds of real number constants:\nlimited precision (:eql:type:`float64`) and unlimited precision\n(:eql:type:`decimal`). The :eql:type:`decimal` constants have the same\nlexical structure as :eql:type:`float64`, but with an ``n`` suffix:\n\n.. productionlist:: edgeql\n decimal: `float` \"n\"\n float: `float_wo_dec` | `float_w_dec`\n float_wo_dec: `integer_part` `exp`\n float_w_dec: `integer_part` \".\" `decimal_part`? `exp`?\n integer_part: \"0\" | `non_zero` `digit`*\n decimal_part: `digit`+\n exp: \"e\" (\"+\" | \"-\")? `digit`+\n\nBy default all float literals are interpreted as :eql:type:`float64`,\nwhile an explicit cast can be used to convert them to :eql:type:`float32`:\n\n.. code-block:: edgeql-repl\n\n db> select 0.1;\n {0.1}\n\n db> select 12.3;\n {12.3}\n\n db> select 1e3;\n {1000.0}\n\n db> select 1.2e-3;\n {0.0012}\n\n db> select 12.3;\n {12.3}\n\nExamples of :eql:type:`decimal` literals:\n\n.. code-block:: edgeql-repl\n\n db> select 12.3n;\n {12.3n}\n\n db> select 12345678901234567890.12345678901234567890n;\n {12345678901234567890.12345678901234567890n}\n\n db> select 12345678901234567890.12345678901234567890e-3n;\n {12345678901234567.89012345678901234567890n}\n\n\nPunctuation\n-----------\n\nEdgeQL uses ``;`` as a statement separator. It is idempotent, so\nmultiple repetitions of ``;`` don't have any additional effect.\n\n\nComments\n--------\n\nComments start with a ``#`` character that is not otherwise part of a\nstring literal and end at the end of line. Semantically, a comment is\nequivalent to whitespace.\n\n.. productionlist:: edgeql\n comment: \"#\" \n\n\nOperators\n---------\n\nEdgeQL operators listed in order of precedence from lowest to highest:\n\n.. list-table::\n :widths: auto\n :header-rows: 1\n\n * - operator\n * - :eql:op:`union`\n * - :eql:op:`if..else`\n * - :eql:op:`or`\n * - :eql:op:`and`\n * - :eql:op:`not`\n * - :eql:op:`=`, :eql:op:`\\!=`, :eql:op:`?=`,\n :eql:op:`?\\!=`\n * - :eql:op:`\\<`, :eql:op:`>`, :eql:op:`\\<=`,\n :eql:op:`>=`\n * - :eql:op:`like`, :eql:op:`ilike`\n * - :eql:op:`in`, :eql:op:`not in `\n * - :eql:op:`is`, :eql:op:`is not `\n * - :eql:op:`+`, :eql:op:`-`, :eql:op:`++`\n * - :eql:op:`*`, :eql:op:`/
`,\n :eql:op:`//`, :eql:op:`%`\n * - :eql:op:`?? `\n * - :eql:op:`distinct`, unary :eql:op:`-`\n * - :eql:op:`^`\n * - :eql:op:`type cast `\n * - :eql:op:`array[] `,\n :eql:op:`str[] `,\n :eql:op:`json[] `,\n :eql:op:`bytes[] `\n * - :eql:kw:`detached`\n" }, { "path": "docs/reference/reference/edgeql/paths.rst", "content": ".. _ref_reference_paths:\n\n=====\nPaths\n=====\n\n\nA *path expression* (or simply a *path*) represents a set of values that are\nreachable when traversing a given sequence of links or properties from some\nsource set.\n\nThe result of a path expression depends on whether it terminates with a link or\nproperty reference.\n\na) if a path *does not* end with a property reference, then it represents a\n unique set of objects reachable from the set at the root of the path;\n\nb) if a path *does* end with a property reference, then it represents a\n list of property values for every element in the unique set of\n objects reachable from the set at the root of the path.\n\nThe syntactic form of a path is:\n\n.. eql:synopsis::\n\n [ ... ]\n\n # where is:\n \n\nThe individual path components are:\n\n:eql:synopsis:``\n Any valid expression.\n\n:eql:synopsis:``\n It can be one of the following:\n\n - ``.`` for an outgoing link reference\n - ``.<`` for an incoming or :ref:`backlink `\n reference\n - ``@`` for a link property reference\n\n:eql:synopsis:``\n This must be a valid link or link property name.\n\n" }, { "path": "docs/reference/reference/edgeql/select.rst", "content": ".. _ref_eql_statements_select:\n\nSelect\n======\n\n:eql-statement:\n:eql-haswith:\n\n``select``--retrieve or compute a set of values.\n\n.. eql:synopsis::\n\n [ with [, ...] ]\n\n select \n\n [ filter ]\n\n [ order by [direction] [then ...] ]\n\n [ offset ]\n\n [ limit ] ;\n\n:eql:synopsis:`filter `\n The optional ``filter`` clause, where :eql:synopsis:``\n is any expression that has a result of type :eql:type:`bool`.\n The condition is evaluated for every element in the set produced by\n the ``select`` clause. The result of the evaluation of the\n ``filter`` clause is a set of boolean values. If at least one value\n in this set is ``true``, the input element is included, otherwise\n it is eliminated from the output.\n\n.. _ref_reference_select_order:\n\n:eql:synopsis:`order by [direction] [then ...]`\n The optional ``order by`` clause has this general form:\n\n .. eql:synopsis::\n\n order by\n [ asc | desc ] [ empty { first | last } ]\n [ then ... ]\n\n The ``order by`` clause produces a result set sorted according\n to the specified expression or expressions, which are evaluated\n for every element of the input set.\n\n If two elements are equal according to the leftmost *expression*, they\n are compared according to the next expression and so on. If two\n elements are equal according to all expressions, the resulting order\n is undefined.\n\n Each *expression* can be an arbitrary expression that results in a\n value of an *orderable type*. Primitive types are orderable,\n object types are not. Additionally, the result of each expression\n must be an empty set or a singleton. Using an expression that may\n produce more elements is a compile-time error.\n\n An optional ``asc`` or ``desc`` keyword can be added after any\n *expression*. If not specified ``asc`` is assumed by default.\n\n If ``empty last`` is specified, then input values that produce\n an empty set when evaluating an *expression* are sorted *after*\n all other values; if ``empty first`` is specified, then they\n are sorted *before* all other values. If neither is specified,\n ``empty first`` is assumed when ``asc`` is specified or implied,\n and ``empty last`` when ``desc`` is specified.\n\n:eql:synopsis:`offset `\n The optional ``offset`` clause, where\n :eql:synopsis:``\n is a *singleton expression* of an integer type.\n This expression is evaluated once and its result is used\n to skip the first *element-count* elements of the input set\n while producing the output. If *element-count* evaluates to\n an empty set, it is equivalent to ``offset 0``, which is equivalent\n to omitting the ``offset`` clause. If *element-count* evaluates\n to a value that is larger then the cardinality of the input set,\n an empty set is produced as the result.\n\n:eql:synopsis:`limit `\n The optional ``limit`` clause, where :eql:synopsis:``\n is a *singleton expression* of an integer\n type. This expression is evaluated once and its result is used\n to include only the first *element-count* elements of the input set\n while producing the output. If *element-count* evaluates to\n an empty set, it is equivalent to specifying no ``limit`` clause.\n\n\nDescription\n-----------\n\n``select`` retrieves or computes a set of values. The data\nflow of a ``select`` block can be conceptualized like this:\n\n.. eql:synopsis::\n\n with module example\n\n # select clause\n select\n # compute a set of things\n\n # optional clause\n filter\n # filter the computed set\n\n # optional clause\n order by\n # define ordering of the filtered set\n\n # optional clause\n offset\n # slice the filtered/ordered set\n\n # optional clause\n limit\n # slice the filtered/ordered set\n\nPlease note that the ``order by`` clause defines ordering that can\nonly be relied upon if the resulting set is not used in any other\noperation. ``select``, ``offset`` and ``limit`` clauses are the only\nexception to that rule as they preserve the inherent ordering of the\nunderlying set.\n\nThe first clause is ``select``. It indicates that ``filter``, ``order\nby``, ``offset``, or ``limit`` clauses may follow an expression, i.e.\nit makes an expression into a ``select`` statement. Without any of the\noptional clauses a ``(select Expr)`` is completely equivalent to\n``Expr`` for any expression ``Expr``.\n\nConsider an example using the ``filter`` optional clause:\n\n.. code-block:: edgeql\n\n with module example\n select User {\n name,\n owned := (select\n User. Select `\n * - :ref:`Cheatsheets > Selecting data `\n" }, { "path": "docs/reference/reference/edgeql/sess_reset_alias.rst", "content": ".. _ref_eql_statements_session_reset_alias:\n\nReset\n=====\n\n:eql-statement:\n\n\n``reset`` -- reset one or multiple session-level parameters\n\n.. eql:synopsis::\n\n reset module ;\n reset alias ;\n reset alias * ;\n reset global ;\n\n\nDescription\n-----------\n\nThis command allows resetting one or many configuration parameters of\nthe current session.\n\n\nVariations\n----------\n\n:eql:synopsis:`reset module`\n Reset the default module name back to \"default\" for the current\n session.\n\n For example, if a module ``foo`` contains type ``FooType``,\n the following is how the ``set`` and ``reset`` commands can be used\n to alias it:\n\n .. code-block:: edgeql\n\n # Set the default module to \"foo\" for the current session.\n set module foo;\n\n # This query is now equivalent to \"select foo::FooType\".\n select FooType;\n\n # Reset the default module for the current session.\n reset module;\n\n # This query will now produce an error.\n select FooType;\n\n\n:eql:synopsis:`reset alias `\n Reset :eql:synopsis:`` for the current session.\n\n For example:\n\n .. code-block:: edgeql\n\n # Alias the \"std\" module as \"foo\".\n set alias foo as module std;\n\n # Now \"std::min()\" can be called as \"foo::min()\" in\n # the current session.\n select foo::min({1});\n\n # Reset the alias.\n reset alias foo;\n\n # Now this query will error out, as there is no\n # module \"foo\".\n select foo::min({1});\n\n:eql:synopsis:`reset alias *`\n Reset all aliases defined in the current session. This command\n affects aliases set with :eql:stmt:`set alias ` and\n :eql:stmt:`set module `. The default module will be set to \"default\".\n\n Example:\n\n .. code-block:: edgeql\n\n # Reset all custom aliases for the current session.\n reset alias *;\n\n\n:eql:synopsis:`reset global `\n Reset the global variable *name* to its default value or ``{}`` if the\n variable has no default value and is ``optional``.\n\n\nExamples\n--------\n\n.. code-block:: edgeql\n\n reset module;\n\n reset alias foo;\n\n reset alias *;\n\n reset global current_user_id;\n\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`Reference > EdgeQL > Set `\n" }, { "path": "docs/reference/reference/edgeql/sess_set_alias.rst", "content": ".. _ref_eql_statements_session_set_alias:\n\nSet\n===\n\n:eql-statement:\n\n\n``set`` -- set one or multiple session-level parameters\n\n.. eql:synopsis::\n\n set module ;\n set alias as module ;\n set global := ;\n\n\nDescription\n-----------\n\nThis command allows altering the configuration of the current session.\n\n\nVariations\n----------\n\n:eql:synopsis:`set module `\n Set the default module for the current section to *module*.\n\n For example, if a module ``foo`` contains type ``FooType``,\n the following is how the type can be referred to:\n\n .. code-block:: edgeql\n\n # Use the fully-qualified name.\n select foo::FooType;\n\n # Use the WITH clause to define the default module\n # for the query.\n with module foo select foo::FooType;\n\n # Set the default module for the current session ...\n set module foo;\n # ... and use an unqualified name.\n select FooType;\n\n\n:eql:synopsis:`set alias as module `\n Define :eql:synopsis:`` for the\n :eql:synopsis:``.\n\n For example:\n\n .. code-block:: edgeql\n\n # Use the fully-qualified name.\n select foo::FooType;\n\n # Use the WITH clause to define a custom alias\n # for the \"foo\" module.\n with bar as module foo\n select bar::FooType;\n\n # Define \"bar\" as an alias for the \"foo\" module for\n # the current session ...\n set alias bar as module foo;\n # ... and use \"bar\" instead of \"foo\".\n select bar::FooType;\n\n\n:eql:synopsis:`set global := `\n Set the global variable *name* to the specified value.\n\n For example:\n\n .. code-block:: edgeql\n\n # Set the global variable \"current_user_id\".\n set global current_user_id :=\n '00ea8eaa-02f9-11ed-a676-6bd11cc6c557';\n\n # We can now use that value in a query.\n select User { name }\n filter .id = global current_user_id;\n\n\nExamples\n--------\n\n.. code-block:: edgeql\n\n set module foo;\n\n set alias foo AS module std;\n\n set global current_user_id :=\n '00ea8eaa-02f9-11ed-a676-6bd11cc6c557';\n\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`Reference > EdgeQL > Reset\n `\n" }, { "path": "docs/reference/reference/edgeql/shapes.rst", "content": ".. _ref_reference_shapes:\n\n======\nShapes\n======\n\nA *shape* is a powerful syntactic construct that can be used to describe\ntype variants in queries, data in ``insert`` and ``update`` statements,\nand to specify the format of statement output.\n\nShapes always follow an expression, and are a list of *shape elements*\nenclosed in curly braces:\n\n.. eql:synopsis::\n\n \"{\"\n [, ...]\n \"}\"\n\n\nShape element has the following syntax:\n\n.. eql:synopsis::\n\n [ \"[\" is \"]\" ] \n\nIf an optional :eql:synopsis:`` filter is used,\n:eql:synopsis:`` will only apply to those objects in\nthe :eql:synopsis:`` set that are instances of\n:eql:synopsis:``.\n\n:eql:synopsis:`` is one of the following:\n\n- a name of an existing link or property of a type produced\n by :eql:synopsis:``;\n\n- a declaration of a computed link or property in the form\n\n .. eql:synopsis ::\n\n [@] := \n\n\n- a *subshape* in the form\n\n .. eql:synopsis ::\n\n : [ \"[\" is \"]\" ] \"{\" ... \"}\"`\n\n The :eql:synopsis:`` is the name of an existing link\n or property, and :eql:synopsis:`` is an optional object\n type that specifies the type of target objects selected or inserted,\n depending on the context.\n\nShaping Query Results\n=====================\n\nAt the end of the day, EdgeQL has two jobs that are similar, yet distinct:\n\n1) Express the values that we want computed.\n2) Arrange the values into a particular shape that we want.\n\nConsider the task of getting \"names of users and all of the friends'\nnames associated with the given user\" in a database defined by the\nfollowing schema:\n\n.. code-block:: sdl\n\n type User {\n required name: str;\n multi friends: User;\n }\n\nIf we only concern ourselves with getting the values, then a\nreasonable solution to this might be:\n\n.. code-block:: edgeql-repl\n\n db> select (User.name, User.friends.name ?? '');\n {\n ('Alice', 'Cameron'),\n ('Alice', 'Dana'),\n ('Billie', 'Dana'),\n ('Cameron', ''),\n ('Dana', 'Alice'),\n ('Dana', 'Billie'),\n ('Dana', 'Cameron'),\n }\n\nThis particular solution is very similar to what one might get using\nSQL. It's equivalent to a table with \"user name\" and \"friend name\"\ncolumns. It gets the job done, albeit with some redundant repeating of\n\"user names\".\n\nWe can improve things a little and reduce the repetition by\naggregating all the friend names into an array:\n\n.. code-block:: edgeql-repl\n\n db> select (User.name, array_agg(User.friends.name));\n {\n ('Alice', ['Cameron', 'Dana']),\n ('Billie', ['Dana']),\n ('Cameron', []),\n ('Dana', ['Alice', 'Billie', 'Cameron']),\n }\n\n\nThis achieves a couple of things: it's easier to see which friends\nbelong to which user and we no longer need the placeholder ``''`` for\nthose users who don't have friends.\n\nThe recommended way to get this information in Gel, however, is to\nuse *shapes*, because they mimic the structure of the data and the output:\n\n.. code-block:: edgeql-repl\n\n db> select User {\n ... name,\n ... friends: {\n ... name\n ... }\n ... };\n {\n default::User {\n name: 'Alice',\n friends: {\n default::User {name: 'Cameron'},\n default::User {name: 'Dana'},\n },\n },\n default::User {name: 'Billie', friends: {default::User {name: 'Dana'}}},\n default::User {name: 'Cameron', friends: {}},\n default::User {\n name: 'Dana',\n friends: {\n default::User {name: 'Alice'},\n default::User {name: 'Billie'},\n default::User {name: 'Cameron'},\n },\n },\n }\n\nSo far the expression for the data that we wanted was also acceptable\nfor structuring the output, but what if that's not the case? Let's add\na condition and only show those users who have friends with either the\nletter \"i\" or \"o\" in their names:\n\n.. code-block:: edgeql-repl\n\n db> select User {\n ... name,\n ... friends: {\n ... name\n ... }\n ... } filter .friends.name ilike '%i%' or .friends.name ilike '%o%';\n {\n default::User {\n name: 'Alice',\n friends: {\n default::User {name: 'Cameron'},\n default::User {name: 'Dana'},\n },\n },\n default::User {\n name: 'Dana',\n friends: {\n default::User {name: 'Alice'},\n default::User {name: 'Billie'},\n default::User {name: 'Cameron'},\n },\n },\n }\n\nThat ``filter`` is getting a bit bulky, so perhaps we can just factor\nthese flags out as part of the shape's computed properties:\n\n.. code-block:: edgeql-repl\n\n db> select User {\n ... name,\n ... friends: {\n ... name\n ... },\n ... has_i := .friends.name ilike '%i%',\n ... has_o := .friends.name ilike '%o%',\n ... } filter .has_i or .has_o;\n {\n default::User {\n name: 'Alice',\n friends: {\n default::User {name: 'Cameron'},\n default::User {name: 'Dana'},\n },\n has_i: {false, false},\n has_o: {true, false},\n },\n default::User {\n name: 'Dana',\n friends: {\n default::User {name: 'Alice'},\n default::User {name: 'Billie'},\n default::User {name: 'Cameron'},\n },\n has_i: {true, true, false},\n has_o: {false, false, true},\n },\n }\n\nIt looks like this refactoring came at the cost of putting extra\nthings into the output. In this case we don't want our intermediate\ncalculations to actually show up in the output, so what can we do? In\n|Gel| the output structure is determined *only* by the expression\nappearing in the top-level :eql:stmt:`select`. This means\nthat we can move our intermediate calculations into the :eql:kw:`with` block:\n\n.. code-block:: edgeql-repl\n\n db> with U := (\n ... select User {\n ... has_i := .friends.name ilike '%i%',\n ... has_o := .friends.name ilike '%o%',\n ... }\n ... )\n ... select U {\n ... name,\n ... friends: {\n ... name\n ... },\n ... } filter .has_i or .has_o;\n {\n default::User {\n name: 'Alice',\n friends: {\n default::User {name: 'Cameron'},\n default::User {name: 'Dana'},\n },\n },\n default::User {\n name: 'Dana',\n friends: {\n default::User {name: 'Alice'},\n default::User {name: 'Billie'},\n default::User {name: 'Cameron'},\n },\n },\n }\n\nThis way we can use ``has_i`` and ``has_o`` in our query without\nleaking them into the output.\n\nGeneral Shaping Rules\n=====================\n\nIn Gel typically all shapes appearing in the top-level\n:eql:stmt:`select` should be reflected in the output. This\nalso applies to shapes no matter where and how they are nested.\nAside from other shapes, this includes nesting in arrays:\n\n.. code-block:: edgeql-repl\n\n db> select array_agg(User {name});\n {\n [\n default::User {name: 'Alice'},\n default::User {name: 'Billie'},\n default::User {name: 'Cameron'},\n default::User {name: 'Dana'},\n ],\n }\n\n... or tuples:\n\n.. code-block:: edgeql-repl\n\n db> select enumerate(User {name});\n {\n (0, default::User {name: 'Alice'}),\n (1, default::User {name: 'Billie'}),\n (2, default::User {name: 'Cameron'}),\n (3, default::User {name: 'Dana'}),\n }\n\nYou can safely access a tuple element and expect the output shape to\nbe intact:\n\n.. code-block:: edgeql-repl\n\n db> select enumerate(User{name}).1;\n {\n default::User {name: 'Alice'},\n default::User {name: 'Billie'},\n default::User {name: 'Cameron'},\n default::User {name: 'Dana'},\n }\n\nAccessing array elements or working with slices also preserves output\nshape and is analogous to using ``offset`` and ``limit`` when working\nwith sets:\n\n.. code-block:: edgeql-repl\n\n db> select array_agg(User {name})[2];\n {default::User {name: 'Cameron'}}\n\n\nLosing Shapes\n=============\n\nThere are some situations where shape information gets completely or\npartially discarded. Any such operation also prevents the altered\nshape from appearing in the output altogether.\n\nIn order for the shape to be preserved, the original expression type\nmust be preserved. This means that :eql:op:`union` can alter the shape,\nbecause the result of a :eql:op:`union` is a :eql:op:`union type\n`. So you can still refer to the common properties, but not to\nthe properties that appeared in the shape.\n\nAs mentioned above, since :eql:op:`union` potentially alters the\nexpression shape it never preserves output shape, even when the\nunderlying type wasn't altered:\n\n.. code-block:: edgeql-repl\n\n db> select User{name} union User{name};\n {\n default::User {id: 7769045a-27bf-11ec-94ea-3f6c0ae59eb3},\n default::User {id: 7b42ed20-27bf-11ec-94ea-7700ec77834e},\n default::User {id: 7fcedbc4-27bf-11ec-94ea-73dcb6f297a4},\n default::User {id: 82f52646-27bf-11ec-94ea-3718ffb8dd15},\n default::User {id: 7769045a-27bf-11ec-94ea-3f6c0ae59eb3},\n default::User {id: 7b42ed20-27bf-11ec-94ea-7700ec77834e},\n default::User {id: 7fcedbc4-27bf-11ec-94ea-73dcb6f297a4},\n default::User {id: 82f52646-27bf-11ec-94ea-3718ffb8dd15},\n }\n\nListing several items inside a set ``{ ... }`` functions identically\nto a :eql:op:`union` and so will also produce a union type and remove\nshape from output.\n\nAnother subtle way for a type union to remove the shape from the output\nis by the :eql:op:`?? ` and the :eql:op:`if..else` operators. Both\nof them determine the result type as the union of the left and right\noperands:\n\n.. code-block:: edgeql-repl\n\n db> select {} ?? User {name};\n {\n default::User {id: 7769045a-27bf-11ec-94ea-3f6c0ae59eb3},\n default::User {id: 7b42ed20-27bf-11ec-94ea-7700ec77834e},\n default::User {id: 7fcedbc4-27bf-11ec-94ea-73dcb6f297a4},\n default::User {id: 82f52646-27bf-11ec-94ea-3718ffb8dd15},\n }\n\nShapes survive array creation (either via :eql:func:`array_agg` or by\nusing ``[ ... ]``), but they follow the same rules as for :eql:op:`union`\nfor array :eql:op:`concatenation `. Basically the element type\nof the resulting array must be a union type and thus all shape\ninformation is lost:\n\n.. code-block:: edgeql-repl\n\n db> select array_agg(User{name}) ++ array_agg(User{name});\n {\n [\n default::User {id: 7769045a-27bf-11ec-94ea-3f6c0ae59eb3},\n default::User {id: 7b42ed20-27bf-11ec-94ea-7700ec77834e},\n default::User {id: 7fcedbc4-27bf-11ec-94ea-73dcb6f297a4},\n default::User {id: 82f52646-27bf-11ec-94ea-3718ffb8dd15},\n default::User {id: 7769045a-27bf-11ec-94ea-3f6c0ae59eb3},\n default::User {id: 7b42ed20-27bf-11ec-94ea-7700ec77834e},\n default::User {id: 7fcedbc4-27bf-11ec-94ea-73dcb6f297a4},\n default::User {id: 82f52646-27bf-11ec-94ea-3718ffb8dd15},\n ],\n }\n\n.. note::\n\n The :eql:stmt:`for` statement preserves the shape given inside the\n ``union`` clause, effectively applying the shape to its entire\n result.\n" }, { "path": "docs/reference/reference/edgeql/tx_commit.rst", "content": "..\n Portions Copyright (c) 2019 MagicStack Inc. and the Gel authors.\n\n Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group\n Portions Copyright (c) 1994, The Regents of the University of California\n\n Permission to use, copy, modify, and distribute this software and its\n documentation for any purpose, without fee, and without a written agreement\n is hereby granted, provided that the above copyright notice and this\n paragraph and the following two paragraphs appear in all copies.\n\n IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR\n DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING\n LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS\n DOCUMENTATION, EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF THE\n POSSIBILITY OF SUCH DAMAGE.\n\n THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES,\n INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY\n AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS\n ON AN \"AS IS\" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATIONS TO\n PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\n\n.. _ref_eql_statements_commit_tx:\n\nCommit\n======\n\n:eql-statement:\n\n\n``commit`` -- commit the current transaction\n\n.. eql:synopsis::\n\n commit ;\n\n\nExample\n-------\n\nCommit the current transaction:\n\n.. code-block:: edgeql\n\n commit;\n\n\nDescription\n-----------\n\nThe ``commit`` command commits the current transaction. All changes\nmade by the transaction become visible to others and are guaranteed to\nbe durable if a crash occurs.\n\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`Reference > EdgeQL > Start transaction\n `\n * - :ref:`Reference > EdgeQL > Rollabck\n `\n * - :ref:`Reference > EdgeQL > Declare savepoint\n `\n * - :ref:`Reference > EdgeQL > Rollback to savepoint\n `\n * - :ref:`Reference > EdgeQL > Release savepoint\n `\n" }, { "path": "docs/reference/reference/edgeql/tx_rollback.rst", "content": "..\n Portions Copyright (c) 2019 MagicStack Inc. and the Gel authors.\n\n Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group\n Portions Copyright (c) 1994, The Regents of the University of California\n\n Permission to use, copy, modify, and distribute this software and its\n documentation for any purpose, without fee, and without a written agreement\n is hereby granted, provided that the above copyright notice and this\n paragraph and the following two paragraphs appear in all copies.\n\n IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR\n DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING\n LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS\n DOCUMENTATION, EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF THE\n POSSIBILITY OF SUCH DAMAGE.\n\n THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES,\n INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY\n AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS\n ON AN \"AS IS\" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATIONS TO\n PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\n\n.. _ref_eql_statements_rollback_tx:\n\nRollback\n========\n\n:eql-statement:\n\n\n``rollback`` -- abort the current transaction\n\n.. eql:synopsis::\n\n rollback ;\n\n\nExample\n-------\n\nAbort the current transaction:\n\n.. code-block:: edgeql\n\n rollback;\n\n\nDescription\n-----------\n\nThe ``rollback`` command rolls back the current transaction and causes all\nupdates made by the transaction to be discarded.\n\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`Reference > EdgeQL > Start transaction\n `\n * - :ref:`Reference > EdgeQL > Commit\n `\n * - :ref:`Reference > EdgeQL > Declare savepoint\n `\n * - :ref:`Reference > EdgeQL > Rollback to savepoint\n `\n * - :ref:`Reference > EdgeQL > Release savepoint\n `\n" }, { "path": "docs/reference/reference/edgeql/tx_sp_declare.rst", "content": "..\n Portions Copyright (c) 2019 MagicStack Inc. and the Gel authors.\n\n Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group\n Portions Copyright (c) 1994, The Regents of the University of California\n\n Permission to use, copy, modify, and distribute this software and its\n documentation for any purpose, without fee, and without a written agreement\n is hereby granted, provided that the above copyright notice and this\n paragraph and the following two paragraphs appear in all copies.\n\n IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR\n DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING\n LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS\n DOCUMENTATION, EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF THE\n POSSIBILITY OF SUCH DAMAGE.\n\n THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES,\n INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY\n AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS\n ON AN \"AS IS\" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATIONS TO\n PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\n\n.. _ref_eql_statements_declare_savepoint:\n\nDeclare savepoint\n=================\n\n:eql-statement:\n\n\n``declare savepoint`` -- declare a savepoint within the current transaction\n\n.. eql:synopsis::\n\n declare savepoint ;\n\n\nDescription\n-----------\n\n``savepoint`` establishes a new savepoint within the current\ntransaction.\n\nA savepoint is a special mark inside a transaction that allows all\ncommands that are executed after it was established to be rolled back,\nrestoring the transaction state to what it was at the time of the\nsavepoint.\n\nIt is an error to declare a savepoint outside of a transaction.\n\n\nExample\n-------\n\n.. code-block:: edgeql\n\n # Will select no objects:\n select test::TestObject { name };\n\n start transaction;\n\n insert test::TestObject { name := 'q1' };\n insert test::TestObject { name := 'q2' };\n\n # Will select two TestObjects with names 'q1' and 'q2'\n select test::TestObject { name };\n\n declare savepoint f1;\n insert test::TestObject { name:='w1' };\n\n # Will select three TestObjects with names\n # 'q1' 'q2', and 'w1'\n select test::TestObject { name };\n rollback to savepoint f1;\n\n # Will select two TestObjects with names 'q1' and 'q2'\n select test::TestObject { name };\n\n rollback;\n\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`Reference > EdgeQL > Start transaction\n `\n * - :ref:`Reference > EdgeQL > Commit\n `\n * - :ref:`Reference > EdgeQL > Rollabck\n `\n * - :ref:`Reference > EdgeQL > Rollback to savepoint\n `\n * - :ref:`Reference > EdgeQL > Release savepoint\n `\n" }, { "path": "docs/reference/reference/edgeql/tx_sp_release.rst", "content": "..\n Portions Copyright (c) 2019 MagicStack Inc. and the Gel authors.\n\n Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group\n Portions Copyright (c) 1994, The Regents of the University of California\n\n Permission to use, copy, modify, and distribute this software and its\n documentation for any purpose, without fee, and without a written agreement\n is hereby granted, provided that the above copyright notice and this\n paragraph and the following two paragraphs appear in all copies.\n\n IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR\n DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING\n LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS\n DOCUMENTATION, EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF THE\n POSSIBILITY OF SUCH DAMAGE.\n\n THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES,\n INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY\n AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS\n ON AN \"AS IS\" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATIONS TO\n PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\n\n.. _ref_eql_statements_release_savepoint:\n\nRelease savepoint\n=================\n\n:eql-statement:\n\n\n``release savepoint`` -- release a previously declared savepoint\n\n.. eql:synopsis::\n\n release savepoint ;\n\n\nDescription\n-----------\n\n``release savepoint`` destroys a savepoint previously defined in the\ncurrent transaction.\n\nDestroying a savepoint makes it unavailable as a rollback point,\nbut it has no other user visible behavior. It does not undo the effects\nof commands executed after the savepoint was established.\n(To do that, see :eql:stmt:`rollback to savepoint`.)\n\n``release savepoint`` also destroys all savepoints that were\nestablished after the named savepoint was established.\n\n\nExample\n-------\n\n.. code-block:: edgeql\n\n start transaction;\n # ...\n declare savepoint f1;\n # ...\n release savepoint f1;\n # ...\n rollback;\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`Reference > EdgeQL > Start transaction\n `\n * - :ref:`Reference > EdgeQL > Commit\n `\n * - :ref:`Reference > EdgeQL > Rollabck\n `\n * - :ref:`Reference > EdgeQL > Declare savepoint\n `\n * - :ref:`Reference > EdgeQL > Rollback to savepoint\n `\n" }, { "path": "docs/reference/reference/edgeql/tx_sp_rollback.rst", "content": "..\n Portions Copyright (c) 2019 MagicStack Inc. and the Gel authors.\n\n Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group\n Portions Copyright (c) 1994, The Regents of the University of California\n\n Permission to use, copy, modify, and distribute this software and its\n documentation for any purpose, without fee, and without a written agreement\n is hereby granted, provided that the above copyright notice and this\n paragraph and the following two paragraphs appear in all copies.\n\n IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR\n DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING\n LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS\n DOCUMENTATION, EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF THE\n POSSIBILITY OF SUCH DAMAGE.\n\n THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES,\n INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY\n AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS\n ON AN \"AS IS\" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATIONS TO\n PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\n\n.. _ref_eql_statements_rollback_savepoint:\n\nRollback to savepoint\n=====================\n\n:eql-statement:\n\n\n``rollback to savepoint`` -- rollback to a savepoint within the current\ntransaction\n\n\n.. eql:synopsis::\n\n rollback to savepoint ;\n\n\nDescription\n-----------\n\nRollback all commands that were executed after the savepoint\nwas established. The savepoint remains valid and can be rolled back\nto again later, if needed.\n\n``rollback to savepoint`` implicitly destroys all savepoints that\nwere established after the named savepoint.\n\n\nExample\n-------\n\n.. code-block:: edgeql\n\n start transaction;\n # ...\n declare savepoint f1;\n # ...\n rollback to savepoint f1;\n # ...\n rollback;\n\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`Reference > EdgeQL > Start transaction\n `\n * - :ref:`Reference > EdgeQL > Commit\n `\n * - :ref:`Reference > EdgeQL > Rollabck\n `\n * - :ref:`Reference > EdgeQL > Declare savepoint\n `\n * - :ref:`Reference > EdgeQL > Release savepoint\n `\n" }, { "path": "docs/reference/reference/edgeql/tx_start.rst", "content": "..\n Portions Copyright (c) 2019 MagicStack Inc. and the Gel authors.\n\n Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group\n Portions Copyright (c) 1994, The Regents of the University of California\n\n Permission to use, copy, modify, and distribute this software and its\n documentation for any purpose, without fee, and without a written agreement\n is hereby granted, provided that the above copyright notice and this\n paragraph and the following two paragraphs appear in all copies.\n\n IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR\n DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING\n LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS\n DOCUMENTATION, EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF THE\n POSSIBILITY OF SUCH DAMAGE.\n\n THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES,\n INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY\n AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS\n ON AN \"AS IS\" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATIONS TO\n PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.\n\n\n.. _ref_eql_statements_start_tx:\n\nStart transaction\n=================\n\n:eql-statement:\n\n\n``start transaction`` -- start a transaction\n\n.. eql:synopsis::\n\n start transaction [ , ... ] ;\n\n # where is one of:\n\n isolation repeatable read\n isolation serializable\n read write | read only\n deferrable | not deferrable\n\n\nDescription\n-----------\n\nThis command starts a new transaction block.\n\nAny Gel command outside of an explicit transaction block starts\nan implicit transaction block; the transaction is then automatically\ncommitted if the command was executed successfully, or automatically\nrollbacked if there was an error. This behavior is often called\n\"autocommit\".\n\nWhen isolation is not specified, it defaults to ``serializable``.\n\nParameters\n----------\n\nThe :eql:synopsis:`` can be one of the following:\n\n:eql:synopsis:`isolation serializable`\n All statements in the current transaction can only see data\n changes that were committed before the first query or data\n modification statement was executed within this transaction.\n If a pattern of reads and writes among concurrent serializable\n transactions creates a situation that could not have occurred\n in any serial (one-at-a-time) execution of those transactions,\n one of them will be rolled back with a serialization failure.\n\n This level is the default isolation level.\n\n Note that, compared to ``repeatable read``, serializable level has a\n significantly higher probability of resulting in serialization failures,\n requires the whole transaction to be retried. If acceptable, consider using\n ``repeatable read`` or\n :ref:`prefer repeatable read `.\n\n:eql:synopsis:`isolation repeatable read`\n All statements in the current transaction can only see data\n changes that were committed before the first query or data\n modification statement was executed within this transaction.\n\n Compared to ``serializable``, this level is less likely to result in\n serialization failures.\n\n It is however possible for this level to allow serialization anomalies.\n This constitutes a series of transactions that would not be allowed if they\n were executed serially instead of concurrently.\n\n For example, assume ``type X { is_selected: bool }`` and following query:\n\n .. code-block:: edgeql\n\n # transaction A: unselect all selected\n update X filter .is_selected set { is_selected := false };\n\n # transaction B: select all unselected\n update X filter not .is_selected set { is_selected := true };\n\n Running these two transactions serially would result in either all ``X``\n being select or none being selected. But if executed concurrently, even with\n the ``repeatable read`` isolation level, we can end up with some ``X`` being\n selected and some not.\n\n To avoid this, we can use the ``serializable`` isolation level.\n\n:eql:synopsis:`read write`\n Sets the transaction access mode to read/write.\n\n This is the default.\n\n:eql:synopsis:`read only`\n Sets the transaction access mode to read-only. Any data\n modifications with :eql:stmt:`insert`, :eql:stmt:`update`, or\n :eql:stmt:`delete` are disallowed. Schema mutations via :ref:`DDL\n ` are also disallowed.\n\n:eql:synopsis:`deferrable`\n The transaction can be set to deferrable mode only when it is\n ``serializable`` and ``read only``. When all three of these\n properties are selected for a transaction, the transaction\n may block when first acquiring its snapshot, after which it is\n able to run without the normal overhead of a ``serializable``\n transaction and without any risk of contributing to or being\n canceled by a serialization failure. This mode is well suited\n for long-running reports or backups.\n\n\nExamples\n--------\n\nStart a new transaction and rollback it:\n\n.. code-block:: edgeql\n\n start transaction;\n select 'Hello World!';\n rollback;\n\nStart a serializable deferrable transaction:\n\n.. code-block:: edgeql\n\n start transaction isolation serializable, read only, deferrable;\n\n\n.. _prefer_repeatable_read:\n\nPrefer repeatable read\n----------------------\n\nIn addition to the isolation levels above, some client libraries also support\n``PreferRepeatableRead`` as a transaction isolation level.\nIn this mode, the server will analyze the query and use ``repeatable read``\nisolation level if it can. When it cannot, it will use ``serializable``\nisolation level.\n\nClient libraries that currently support this mode:\n\n* TypeScript/JS\n* Python\n* Go\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`Reference > EdgeQL > Commit\n `\n * - :ref:`Reference > EdgeQL > Rollback\n `\n * - :ref:`Reference > EdgeQL > Declare savepoint\n `\n * - :ref:`Reference > EdgeQL > Rollback to savepoint\n `\n * - :ref:`Reference > EdgeQL > Release savepoint\n `\n" }, { "path": "docs/reference/reference/edgeql/update.rst", "content": ".. _ref_eql_statements_update:\n\nUpdate\n======\n\n:eql-statement:\n:eql-haswith:\n\n``update`` -- update objects in a database\n\n.. eql:synopsis::\n\n [ with [, ...] ]\n\n update \n\n [ filter ]\n\n set ;\n\n``update`` changes the values of the specified links in all objects\nselected by *update-selector-expr* and, optionally, filtered by\n*filter-expr*.\n\n:eql:synopsis:`with`\n Alias declarations.\n\n The ``with`` clause allows specifying module aliases as well\n as expression aliases that can be referenced by the ``update``\n statement. See :ref:`ref_eql_statements_with` for more information.\n\n:eql:synopsis:`update `\n An arbitrary expression returning a set of objects to be updated.\n\n:eql:synopsis:`filter `\n An expression of type :eql:type:`bool` used to filter the\n set of updated objects.\n\n :eql:synopsis:`` is an expression that has a result\n of type :eql:type:`bool`. Only objects that satisfy the filter\n expression will be updated. See the description of the\n ``filter`` clause of the :eql:stmt:`select` statement for more\n information.\n\n:eql:synopsis:`set `\n A shape expression with the\n new values for the links of the updated object. There are three\n possible assignment operations permitted within the ``set`` shape:\n\n .. eql:synopsis::\n\n set { := [, ...] }\n\n set { += [, ...] }\n\n set { -= [, ...] }\n\n The most basic assignment is the ``:=``, which just sets the\n :eql:synopsis:`` to the specified\n :eql:synopsis:``. The ``+=`` and ``-=`` either add or\n remove the set of values specified by the\n :eql:synopsis:`` from the *current* value of the\n :eql:synopsis:``.\n\nOutput\n~~~~~~\n\nOn successful completion, an ``update`` statement returns the\nset of updated objects.\n\n\nExamples\n~~~~~~~~\n\nHere are a couple of examples of the ``update`` statement with simple\nassignments using ``:=``:\n\n.. code-block:: edgeql\n\n # update the user with the name 'Alice Smith'\n with module example\n update User\n filter .name = 'Alice Smith'\n set {\n name := 'Alice J. Smith'\n };\n\n # update all users whose name is 'Bob'\n with module example\n update User\n filter .name like 'Bob%'\n set {\n name := User.name ++ '*'\n };\n\nFor usage of ``+=`` and ``-=`` consider the following ``Post`` type:\n\n.. code-block:: sdl\n\n # ... Assume some User type is already defined\n type Post {\n required title: str;\n required body: str;\n # A \"tags\" property containing a set of strings\n multi tags: str;\n author: User;\n }\n\nThe following queries add or remove tags from some user's posts:\n\n.. code-block:: edgeql\n\n with module example\n update Post\n filter .author.name = 'Alice Smith'\n set {\n # add tags\n tags += {'example', 'edgeql'}\n };\n\n with module example\n update Post\n filter .author.name = 'Alice Smith'\n set {\n # remove a tag, if it exist\n tags -= 'todo'\n };\n\n\nThe statement ``for in `` allows to express certain bulk\nupdates more clearly. See\n:ref:`ref_eql_forstatement` for more details.\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`EdgeQL > Update `\n * - :ref:`Cheatsheets > Updating data `\n" }, { "path": "docs/reference/reference/edgeql/volatility.rst", "content": ".. _ref_reference_volatility:\n\n\nVolatility\n==========\n\nThe **volatility** of an expression refers to how its value may change across\nsuccessive evaluations. \n\nExpressions may have one of the following volatilities, in order of increasing\nvolatility:\n\n* ``Immutable``: The expression cannot modify the database and is\n guaranteed to have the same value *in all statements*.\n\n* ``Stable``: The expression cannot modify the database and is\n guaranteed to have the same value *within a single statement*.\n\n* ``Volatile``: The expression cannot modify the database and can have\n different values on successive evaluations.\n\n* ``Modifying``: The expression can modify the database and can have\n different values on successive evaluations.\n\n\nExpressions\n-----------\n\nAll :ref:`primitives `,\n:ref:`ranges `, and\n:ref:`multiranges ` are ``Immutable``.\n\n:ref:`Arrays `, :ref:`tuples `, and\n:ref:`sets ` have the volatility of their most volatile\ncomponent.\n\n:ref:`Globals ` are always ``Stable``, even computed\nglobals with an immutable expression.\n\n\nObjects and shapes\n^^^^^^^^^^^^^^^^^^\n\n:ref:`Objects ` are generally ``Stable`` except:\n\n* Objects with a :ref:`shape ` containing a more volatile\n computed pointer will have the volatility of its most volatile component.\n\n* :ref:`Free objects ` have the volatility of\n their most volatile component. They may be ``Immutable``.\n\nAn object's non-computed pointers are ``Stable``. Its computed pointers have\nthe volatility of their expressions.\n\nAny DML (i.e., :ref:`insert `, :ref:`update `,\n:ref:`delete `) is ``Modifying``.\n\n\nFunctions and operators\n^^^^^^^^^^^^^^^^^^^^^^^\n\nUnless explicitly specified, a :ref:`function's `\nvolatility will be inferred from its body expression.\n\nA function call's volatility is highest of its body expression and its call\narguments.\n\nGiven:\n\n.. code-block:: sdl\n\n # Immutable\n function plus_primitive(x: float64) -> float64\n using (x + 1);\n\n # Stable\n global one := 1;\n function plus_global(x: float64) -> float64\n using (x + one);\n\n # Volatile\n function plus_random(x: float64) -> float64\n using (x + random());\n\n # Modifying\n type One {\n val := 1;\n };\n function plus_insert(x: float64) -> float64\n using (x + (insert One).val);\n\nSome example operator and function calls:\n\n.. code-block::\n\n 1 + 1: Immutable\n 1 + global one: Stable\n global one + random(): Volatile\n (insert One).val: Modifying\n plus_primitive(1): Immutable\n plus_stable(1): Stable\n plus_random(global one): Volatile\n plus_insert(random()): Immutable\n\n\nRestrictions\n------------\n\nSome features restrict the volatility of expressions. A lower volatility\ncan be used.\n\n:ref:`Indexes ` expressions must be ``Immutable``.\nWithin the index, pointers to the indexed object are treated as immutable\n\n:ref:`constraints ` expressions must be\n``Immutable``. Within the constraint, the ``__subject__`` and its pointers are\ntreated as immutable.\n\n:ref:`Access policies ` must be ``Stable``.\n\n:ref:`Aliases `, :ref:`globals `,\nand :ref:`computed pointers ` in the schema must be\n``Stable``.\n\nThe :ref:`cartesian product ` of a\n``Volatile`` or ``Modifying`` expression is not allowed.\n\n.. code-block:: edgeql-repl\n\n db> SELECT {1, 2} + random()\n QueryError: can not take cross product of volatile operation\n\n``Modifying`` expressions are not allowed in a non-scalar argument to a\nfunction, except for :ref:`standard set functions `.\n\nThe non-optional parameters of ``Modifying``\n:ref:`functions ` must have a\n:ref:`cardinality ` of ``One``. Optional\nparameters must have a cardinality of ``AtMostOne``.\n" }, { "path": "docs/reference/reference/edgeql/with.rst", "content": ".. _ref_eql_statements_with:\n\nWith block\n==========\n\n.. eql:keyword:: with\n\n The ``with`` block in EdgeQL is used to define aliases.\n\n The expression aliases are evaluated in the lexical scope they appear in,\n not the scope where their alias is used. This means that refactoring\n queries using aliases must be done with care so as not to alter the query\n semantics.\n\nSpecifying a module\n+++++++++++++++++++\n\n.. eql:keyword:: module\n\n Used inside a ``with`` block to specify module names.\n\nOne of the more basic and common uses of the ``with`` block is to\nspecify the default module that is used in a query. ``with module\n`` construct indicates that whenever an identifier is used\nwithout any module specified explicitly, the module will default to\n```` and then fall back to built-ins from ``std`` module.\n\nThe following queries are exactly equivalent:\n\n.. code-block:: edgeql\n\n with module example\n select User {\n name,\n owned := (select\n User. With `\n" }, { "path": "docs/reference/reference/index.rst", "content": ".. versioned-section::\n\n.. _ref_reference_index:\n\n=========\nReference\n=========\n\n.. toctree::\n :maxdepth: 3\n :hidden:\n\n edgeql/index\n\n\nThis section contains comprehensive reference documentation on the internals of\n|Gel|, the binary protocol, the formal syntax of EdgeQL, and more.\n" }, { "path": "docs/reference/running/admin/configure.rst", "content": ".. _ref_eql_statements_configure:\n\nConfigure\n=========\n\n:eql-statement:\n\n\n``configure`` -- change a server configuration parameter\n\n.. eql:synopsis::\n\n configure {session | current branch | instance}\n set := ;\n configure instance insert ;\n configure {session | current branch | instance} reset ;\n configure {current branch | instance}\n reset [ filter ] ;\n\n.. note::\n Prior to |Gel| and |EdgeDB| 5.0 *branches* were called *databases*.\n ``configure current branch`` is used to be called\n ``configure current database``, which is still supported for backwards\n compatibility.\n\n\nDescription\n-----------\n\nThis command allows altering the server configuration.\n\nThe effects of :eql:synopsis:`configure session` last until the end of the\ncurrent session. Some configuration parameters cannot be modified by\n:eql:synopsis:`configure session` and can only be set by\n:eql:synopsis:`configure instance`.\n\n:eql:synopsis:`configure current branch` is used to configure an\nindividual Gel branch within a server instance with the\nchanges persisted across server restarts.\n\n:eql:synopsis:`configure instance` is used to configure the entire Gel\ninstance with the changes persisted across server restarts. This variant\nacts directly on the file system and cannot be rolled back, so it cannot\nbe used in a transaction block.\n\nThe :eql:synopsis:`configure instance insert` variant is used for composite\nconfiguration parameters, such as ``Auth``.\n\n\nParameters\n----------\n\n:eql:synopsis:``\n The name of a primitive configuration parameter. Available\n configuration parameters are described in the :ref:`ref_std_cfg`\n section.\n\n:eql:synopsis:``\n The name of a composite configuration value class. Available\n configuration classes are described in the :ref:`ref_std_cfg`\n section.\n\n:eql:synopsis:``\n An expression that returns a value of type :eql:type:`std::bool`.\n Only configuration objects matching this condition will be affected.\n\n\nExamples\n--------\n\nSet the ``listen_addresses`` parameter:\n\n.. code-block:: edgeql\n\n configure instance set listen_addresses := {'127.0.0.1', '::1'};\n\nSet the ``query_work_mem`` parameter:\n\n.. code-block:: edgeql\n\n configure instance set query_work_mem := '4MiB';\n\nAdd a Trust authentication method for \"my_user\":\n\n.. code-block:: edgeql\n\n configure instance insert Auth {\n priority := 1,\n method := (insert Trust),\n user := 'my_user'\n };\n\nRemove all Trust authentication methods:\n\n.. code-block:: edgeql\n\n configure instance reset Auth filter Auth.method is Trust;\n" }, { "path": "docs/reference/running/admin/index.rst", "content": ".. _ref_admin:\n\nAdministration\n==============\n\nAdministrative commands for managing Gel:\n\n\n* :ref:`configure `\n\n Configure server behavior.\n\n* :ref:`role `\n\n Create, remove, or alter a role.\n\n.. versionadded:: 5.0\n\n New administrative commands were added in |EdgeDB| 5 release:\n\n * :ref:`branch `\n\n Create, remove, or alter a branch.\n\n * :ref:`administer statistics_update() `\n\n Update internal statistics about data.\n\n * :ref:`administer vacuum() `\n\n Reclaim storage space.\n\n\n.. toctree::\n :maxdepth: 3\n :hidden:\n\n configure\n roles\n statistics_update\n vacuum\n" }, { "path": "docs/reference/running/admin/roles.rst", "content": ".. _ref_admin_roles:\n\n=====\nRoles\n=====\n\n:edb-alt-title: Roles\n\n\nThis section describes the administrative commands pertaining to *roles*.\n\n\nCreate role\n===========\n\n:eql-statement:\n\nCreate a role.\n\n.. eql:synopsis::\n\n create superuser role [ extending [, ...] ]\n \"{\" ; [...] \"}\" ;\n\n # where is one of\n\n set password := \n\n\nDescription\n-----------\n\nThe command ``create role`` defines a new database role.\n\n:eql:synopsis:`superuser`\n If specified, the created role will have the *superuser* status, and\n will be exempt from\n :ref:`all permission checks`.\n\n Prior to version 7.0, ``superuser`` qualifier was mandatory, i.e. it was not\n possible to create non-superuser roles.\n\n:eql:synopsis:``\n The name of the role to create.\n\n:eql:synopsis:`extending [, ...]`\n If specified, declares the parent roles for this role. The role\n inherits all the privileges of the parents.\n\nThe following subcommands are allowed in the ``create role`` block:\n\n:eql:synopsis:`set password := `\n Set the password for the role.\n\n.. versionadded:: 7.0\n\n :eql:synopsis:`set permissions := `\n Set :ref:`permissions ` for the role.\n Value is a set of identifiers of either built-in permissions or\n permissions defined in schema.\n\n Roles also gain the permissions of their base Roles. \n\n Roles that are *superusers* are implicitly granted all permissions, so\n setting this does not have any effect.\n\n Note that permission names are not validated and it is possible to\n reference a permission that does not yet exist in any schema.\n\n :eql:synopsis:`set branches := `\n Configure a set of branches that this role is allowed to access.\n When connecting to instance branch that is not in this set, connection\n will be refused.\n\n If set to ``'*'``, this branch can connect to all branches of the\n instance. Defaults to ``'*'``.\n\n\nExamples\n--------\n\nCreate a new role:\n\n.. code-block:: edgeql\n\n create role alice {\n set password := 'wonderland';\n set permissions := {\n sys::perm::data_modifiction,\n sys::perm::query_stats,\n cfg::perm::configure_timeouts,\n cfg::perm::configure_apply_access_policies,\n ext::auth::perm::auth_read,\n ext::auth::perm::auth_write,\n };\n set branches := {'main', 'staging'};\n };\n\n\nAlter role\n==========\n\n:eql-statement:\n\nAlter an existing role.\n\n.. eql:synopsis::\n\n alter role \"{\" ; [...] \"}\" ;\n\n # where is one of\n\n rename to \n set password := \n extending ...\n\n\nDescription\n-----------\n\nThe command ``alter role`` changes the settings of an existing role.\n\n\n:eql:synopsis:``\n The name of the role to alter.\n\nThe following subcommands are allowed in the ``alter role`` block:\n\n:eql:synopsis:`rename to `\n Change the name of the role to *newname*.\n\n:eql:synopsis:`extending ...`\n Alter the role parent list. The full syntax of this subcommand is:\n\n .. eql:synopsis::\n\n extending [, ...]\n [ first | last | before | after ]\n\n This subcommand makes the role a child of the specified list of\n parent roles. The role inherits all the privileges of the parents.\n\n It is possible to specify the position in the parent list\n using the following optional keywords:\n\n * ``first`` -- insert parent(s) at the beginning of the\n parent list,\n * ``last`` -- insert parent(s) at the end of the parent list,\n * ``before `` -- insert parent(s) before an\n existing *parent*,\n * ``after `` -- insert parent(s) after an existing\n *parent*.\n\n.. versionadded:: 7.0\n\n :eql:synopsis:`set permissions := `\n Set :ref:`permissions ` for the role.\n Value is a set of identifiers of either built-in permissions or\n permissions defined in schema.\n\n Roles that are *superusers* are implicitly granted all permissions, so\n setting this does not have any effect.\n\n Note that permission names are not validated and it is possible to\n reference a permission that does not yet exist in the schema.\n\n :eql:synopsis:`set branches := `\n Configure a set of branches that this role is allowed to access.\n When connecting to instance branch that is not in this set, connection\n will be refused.\n\n If set to ``'*'``, this branch can connect to all branches of the\n instance. Defaults to ``'*'``.\n\n\nExamples\n--------\n\nAlter a role:\n\n.. code-block:: edgeql\n\n alter role alice {\n set password := 'new password';\n set branches := {'*'};\n };\n\n\nDrop role\n=========\n\n:eql-statement:\n\nRemove a role.\n\n.. eql:synopsis::\n\n drop role ;\n\nDescription\n-----------\n\nThe command ``drop role`` removes an existing role.\n\nExamples\n--------\n\nRemove a role:\n\n.. code-block:: edgeql\n\n drop role alice;\n" }, { "path": "docs/reference/running/admin/statistics_update.rst", "content": ".. versionadded:: 6.0\n\n.. _ref_admin_statistics_update:\n\n==============================\nadminister statistics_update()\n==============================\n\n:eql-statement:\n\nUpdate internal statistics about data.\n\n.. eql:synopsis::\n\n administer statistics_update \"(\"\n [ [, ...]]\n \")\"\n\n\nDescription\n-----------\n\nUpdates statistics about the contents of data in the current branch.\nSubsequently, the query planner uses these statistics to help determine the\nmost efficient execution plans for queries.\n\n:eql:synopsis:``\n If a type name or a path to a link or property are specified, that data\n will be targeted for statistics update. If omitted, all user-accessible\n data will be analyzed.\n\n\nExamples\n--------\n\nUpdate the statistics on type ``SomeType``:\n\n.. code-block:: edgeql\n\n administer statistics_update(SomeType);\n\nUpdate statistics of type ``SomeType`` and the link ``OtherType.ptr``.\n\n.. code-block:: edgeql\n\n administer statistics_update(SomeType, OtherType.ptr);\n\nUpdate statistics on everything that is user-accessible in the database:\n\n.. code-block:: edgeql\n\n administer statistics_update();\n" }, { "path": "docs/reference/running/admin/vacuum.rst", "content": ".. versionadded:: 5.0\n\n.. _ref_admin_vacuum:\n\n======\nVacuum\n======\n\n:eql-statement:\n\nReclaim storage space.\n\n.. eql:synopsis::\n\n administer vacuum \"(\"\n [ [, ...]]\n [, full := {true | false}]\n [, statistics_update := {true | false}]\n \")\"\n\n\nDescription\n-----------\n\nCleans and reclaims storage by removing obsolete data.\n\n:eql:synopsis:``\n If a type name or a path to a link or property are specified, that data\n will be targeted for the vacuum operation. If omitted, all user-accessible\n data will be targeted.\n\n:eql:synopsis:`full := {true | false}`\n If set to ``true``, an exclusive lock is obtained and reclaimed space is\n returned to the operating system. If set to ``false`` or if not set, the\n command can operate alongside normal reading and writing of the database\n and reclaimed space is kept available for reuse in the database, reducing\n the rate of growth of the database.\n\n:eql:synopsis:`statistics_update := {true | false}`\n If set to ``true``, updates statistics used by the planner to determine\n the most efficient way to execute queries on specified data. See also\n :ref:`administer statistics_update() `.\n\nExamples\n--------\n\nVacuum the type ``SomeType``:\n\n.. code-block:: edgeql\n\n administer vacuum(SomeType);\n\nVacuum the type ``SomeType`` and the link ``OtherType.ptr`` and return\nreclaimed space to the operating system:\n\n.. code-block:: edgeql\n\n administer vacuum(SomeType, OtherType.ptr, full := true);\n\nVacuum everything that is user-accessible in the database:\n\n.. code-block:: edgeql\n\n administer vacuum();\n" }, { "path": "docs/reference/running/backend_ha.rst", "content": ".. _ref_backend_ha:\n\nBackend high-availability\n=========================\n\nHigh availability is a sophisticated and systematic challenge, especially for\ndatabases. To address the problem, Gel server now supports selected\nhighly-available backend Postgres clusters, namely in 2 categories:\n\n* API-based HA\n* Adaptive HA without API\n\nWhen the backend HA feature is enabled in Gel, Gel server will try its\nbest to detect and react to backend failovers, whether a proper API is\navailable or not.\n\nDuring backend failover, no frontend connections will be closed; instead, all\nincoming queries will fail with a retryable error until failover has completed\nsuccessfully. If the query originates from a client that supports retrying\ntransactions, these queries may be retried by the client until the backend\nconnection is restored and the query can be properly resolved.\n\nAPI-based HA\n------------\n\n|Gel| server accepts different types of backends by looking into the protocol\nof the ``--backend-dsn`` command-line parameter. Gel supports the following\nDSN protocols currently:\n\n* ``stolon+consul+http://``\n* ``stolon+consul+https://``\n\nWhen using these protocols, Gel builds the actual DSN of the cluster's\nleader node by calling the corresponding API using credentials in the\n``--backend-dsn`` and subscribes to that API for failover events. Once failover\nis detected, Gel drops all backend connections and routes all new backend\nconnections to the new leader node.\n\n`Stolon `_ is an open-source cloud native\nPostgreSQL manager for PostgreSQL high availability. Currently, Gel supports\nusing a Stolon cluster as the backend in a Consul-based setup, where Gel\nacts as a Stolon proxy. This way, you only need to manage Stolon sentinels and\nkeepers, plus a Consul deployment. To use a Stolon cluster, run Gel server\nwith a DSN, like so:\n\n.. code-block:: bash\n\n $ gel-server \\\n --backend-dsn stolon+consul+http://localhost:8500/my-cluster\n\n|Gel| will connect to the Consul HTTP service at ``localhost:8500``, and\nsubscribe to the updates of the cluster named ``my-cluster``.\n\nUsing a regular ``postgres://`` DSN disables API-based HA.\n\n\nAdaptive HA\n-----------\n\n|Gel| also supports DNS-based generic HA backends. This may be a cloud\ndatabase with multi-AZ failover or some custom HA Postgres cluster that keeps\na DNS name always resolved to the leader node. Adaptive HA can be enabled with\na switch in addition to a regular backend DSN:\n\n.. code-block:: bash\n\n $ gel-server \\\n --backend-dsn postgres://xxx.rds.amazonaws.com \\\n --enable-backend-adaptive-ha\n\nOnce enabled, Gel server will keep track of unusual backend events like\nunexpected disconnects or Postgres shutdown notifications. When a threshold is\nreached, Gel considers the backend to be in the \"failover\" state. It then\ndrops all current backend connections and try to re-establish new connections\nwith the same backend DSN. Because Gel doesn't cache resolved DNS values,\nthe new connections will be established with the new leader node.\n\nUnder the hood of adaptive HA, Gel maintains a state machine to avoid\nendless switch-overs in an unstable network. State changes only happen when\ncertain conditions are met.\n\n**Set of possible states:**\n\n* ``Healthy`` - all is good\n* ``Unhealthy`` - a staging state before failover\n* ``Failover`` - backend failover is in process\n\n**Rules of state switches:**\n\n``Unhealthy`` -> ``Healthy``\n\n* Successfully connected to a non-hot-standby backend.\n\n``Unhealthy`` -> ``Failover``\n\n* More than 60% (configurable with environment variable\n :gelenv:`SERVER_BACKEND_ADAPTIVE_HA_DISCONNECT_PERCENT`) of existing pgcons\n are \"unexpectedly disconnected\" (number of existing pgcons is captured at the\n moment we change to ``Unhealthy`` state, and maintained on \"expected\n disconnects\" too).\n* (and) In ``Unhealthy`` state for more than 30 seconds\n (:gelenv:`SERVER_BACKEND_ADAPTIVE_HA_UNHEALTHY_MIN_TIME`).\n* (and) sys_pgcon is down.\n* (or) Postgres shutdown/hot-standby notification received.\n\n``Healthy`` -> ``Unhealthy``\n\n* Any unexpected disconnect.\n\n``Healthy`` -> ``Failover``\n\n* Postgres shutdown/hot-standby notification received.\n\n``Failover`` -> ``Healthy``\n\n* Successfully connected to a non-hot-standby backend.\n* (and) sys_pgcon is healthy.\n\n(\"pgcon\" is a code name for backend connections, and \"sys_pgcon\" is a special\nbackend connection which Gel uses to talk to the \"Gel system database\".)\n" }, { "path": "docs/reference/running/configuration.rst", "content": ".. _ref_admin_config:\n\n=============\nConfiguration\n=============\n\nThe behavior of the Gel server is configurable with sensible defaults. Some configuration can be set on the running instance using configuration parameters, while other configuration is set at startup using environment variables or command line arguments to the |gel-server| binary.\n\nConfiguration parameters\n========================\n\n|Gel| exposes a number of configuration parameters that affect its behavior. In this section we review the ways to change the server configuration, as well as detail each available configuration parameter.\n\nEdgeQL\n------\n\nThe :eql:stmt:`configure` command can be used to set the configuration parameters using EdgeQL. For example, you can use the CLI REPL to set the ``listen_addresses`` parameter:\n\n.. code-block:: edgeql-repl\n\n gel> configure instance set listen_addresses := {'127.0.0.1', '::1'};\n CONFIGURE: OK\n\nCLI\n---\n\nThe :ref:`ref_cli_gel_configure` command allows modifying the system configuration from a terminal or a script:\n\n.. code-block:: bash\n\n $ gel configure set listen_addresses 127.0.0.1 ::1\n\n\nConfiguration parameters\n========================\n\n:edb-alt-title: Available Configuration Parameters\n\n.. _ref_admin_config_connection:\n\nConnection settings\n-------------------\n\n.. api-index:: listen_addresses, listen_port, cors_allow_origins\n\n:eql:synopsis:`listen_addresses: multi str`\n Specifies the TCP/IP address(es) on which the server is to listen for connections from client applications. If the list is empty, the server does not listen on any IP interface at all.\n\n:eql:synopsis:`listen_port: int16`\n The TCP port the server listens on; ``5656`` by default. Note that the same port number is used for all IP addresses the server listens on.\n\n:eql:synopsis:`cors_allow_origins: multi str`\n Origins that will be calling the server that need Cross-Origin Resource Sharing (CORS) support. Can use ``*`` to allow any origin. When HTTP clients make a preflight request to the server, the origins allowed here will be added to the ``Access-Control-Allow-Origin`` header in the response.\n\nResource usage\n--------------\n\n.. api-index:: effective_io_concurrency, query_work_mem, shared_buffers\n\n:eql:synopsis:`effective_io_concurrency: int64`\n Sets the number of concurrent disk I/O operations that can be executed simultaneously. Corresponds to the PostgreSQL configuration parameter of the same name.\n\n:eql:synopsis:`query_work_mem: cfg::memory`\n The amount of memory used by internal query operations such as sorting. Corresponds to the PostgreSQL ``work_mem`` configuration parameter.\n\n:eql:synopsis:`shared_buffers: cfg::memory`\n The amount of memory the database uses for shared memory buffers. Corresponds to the PostgreSQL configuration parameter of the same name. Changing this value requires server restart.\n\n\nQuery planning\n--------------\n\n.. api-index:: default_statistics_target, effective_cache_size\n\n:eql:synopsis:`default_statistics_target: int64`\n Sets the default data statistics target for the planner. Corresponds to the PostgreSQL configuration parameter of the same name.\n\n:eql:synopsis:`effective_cache_size: cfg::memory`\n Sets the planner's assumption about the effective size of the disk cache that is available to a single query. Corresponds to the PostgreSQL configuration parameter of the same name.\n\n\nQuery cache\n-----------\n\n.. versionadded:: 5.0\n\n.. api-index:: auto_rebuild_query_cache, query_cache_mode, cfg::QueryCacheMode\n\n:eql:synopsis:`auto_rebuild_query_cache: bool`\n Determines whether to recompile the existing query cache to SQL any time DDL is executed.\n\n:eql:synopsis:`query_cache_mode: cfg::QueryCacheMode`\n Allows the developer to set where the query cache is stored. Possible values:\n\n * ``cfg::QueryCacheMode.InMemory``- All query cache is lost on server restart. This mirrors pre-5.0 |EdgeDB| behavior.\n * ``cfg::QueryCacheMode.RegInline``- The in-memory query cache is also stored in the database as-is so it can be restored on restart.\n * ``cfg::QueryCacheMode.Default``- Allow the server to select the best caching option. Currently, it will select ``InMemory`` for arm64 Linux and ``RegInline`` for everything else.\n * ``cfg::QueryCacheMode.PgFunc``- Wraps queries into stored functions in Postgres and reduces backend request size and preparation time.\n\nQuery behavior\n--------------\n\n.. api-index:: allow_bare_ddl, cfg::AllowBareDDL, apply_access_policies,\n apply_access_policies_pg, force_database_error\n\n:eql:synopsis:`allow_bare_ddl: cfg::AllowBareDDL`\n Allows for running bare DDL outside a migration. Possible values are ``cfg::AllowBareDDL.AlwaysAllow`` and ``cfg::AllowBareDDL.NeverAllow``.\n\n When you create an instance, this is set to ``cfg::AllowBareDDL.AlwaysAllow`` until you run a migration. At that point it is set to ``cfg::AllowBareDDL.NeverAllow`` because it's generally a bad idea to mix migrations with bare DDL.\n\n.. _ref_std_cfg_apply_access_policies:\n\n:eql:synopsis:`apply_access_policies: bool`\n Determines whether access policies should be applied when running queries. Setting this to ``false`` effectively puts you into super-user mode, ignoring any access policies that might otherwise limit you on the instance.\n\n .. note::\n\n This setting can also be conveniently accessed via the \"Config\" dropdown menu at the top of the Gel UI (accessible by running the CLI command :gelcmd:`ui` from within a project). The setting will apply only to your UI session, so you won't have to remember to re-enable it when you're done.\n\n:eql:synopsis:`apply_access_policies_pg -> bool`\n Determines whether access policies should be applied when running queries over SQL adapter. Defaults to ``false``.\n\n:eql:synopsis:`force_database_error -> str`\n A hook to force all queries to produce an error. Defaults to 'false'.\n\n .. note::\n\n This parameter takes a ``str`` instead of a ``bool`` to allow more verbose messages when all queries are forced to fail. The database will attempt to deserialize this ``str`` into a JSON object that must include a ``type`` (which must be a Gel :ref:`error type ` name), and may also include ``message``, ``hint``, and ``details`` which can be set ad-hoc by the user.\n\n For example, the following is valid input:\n\n ``'{ \"type\": \"QueryError\",\n \"message\": \"Did not work\",\n \"hint\": \"Try doing something else\",\n \"details\": \"Indeed, something went really wrong\" }'``\n\n As is this:\n\n ``'{ \"type\": \"UnknownParameterError\" }'``\n\nTransaction behavior\n--------------------\n\n.. api-index:: default_transaction_isolation, default_transaction_access_mode, default_transaction_deferrable\n\n.. versionadded:: 6.0\n\nThese settings will affect both explicit transactions as well as the implicit transactions that each query runs in.\n\n:eql:synopsis:`default_transaction_isolation -> sys::TransactionIsolation`\n Controls the default isolation level of each new transaction, including implicit transactions. Defaults to ``sys::TransactionIsolation.Serializable``.\n\n * ``sys::TransactionIsolation.RepeatableRead``\n * ``sys::TransactionIsolation.Serializable`` (default)\n\n:eql:synopsis:`default_transaction_access_mode -> sys::TransactionAccessMode`\n Controls the default read-only status of each new transaction, including implicit transactions. Defaults to ``sys::TransactionAccessMode.ReadWrite``.\n\n * ``sys::TransactionAccessMode.ReadOnly``\n * ``sys::TransactionAccessMode.ReadWrite`` (default)\n\n:eql:synopsis:`default_transaction_deferrable -> sys::TransactionDeferrability`\n Controls the default deferrable status of each new transaction. It currently has no effect on read-write transactions. Defaults to ``sys::TransactionDeferrability.NotDeferrable``.\n\n * ``sys::TransactionDeferrability.Deferrable``\n * ``sys::TransactionDeferrability.NotDeferrable`` (default)\n\n.. _ref_std_cfg_client_connections:\n\nClient connections\n------------------\n\n.. api-index:: allow_user_specified_id, session_idle_timeout,\n session_idle_transaction_timeout, query_execution_timeout\n\n:eql:synopsis:`allow_user_specified_id: bool`\n Makes it possible to set the ``.id`` property when inserting new objects.\n\n .. warning::\n\n Enabling this feature introduces some security vulnerabilities:\n\n 1. An unprivileged user can discover ids that already exist in the database by trying to insert new values and noting when there is a constraint violation on ``.id`` even if the user doesn't have access to the relevant table.\n\n 2. It allows re-using object ids for a different object type, which the application might not expect.\n\n Additionally, enabling can have serious performance implications as, on an ``insert``, every object type must be checked for collisions.\n\n As a result, we don't recommend enabling this. If you need to preserve UUIDs from an external source on your objects, it's best to create a new property to store these UUIDs. If you will need to filter on this external UUID property, you may add an :ref:`index ` or exclusive constraint on it.\n\n:eql:synopsis:`session_idle_timeout -> std::duration`\n Sets the timeout for how long client connections can stay inactive before being forcefully closed by the server.\n\n Time spent on waiting for query results doesn't count as idling. E.g. if the session idle timeout is set to 1 minute it would be OK to run a query that takes 2 minutes to compute; to limit the query execution time use the ``query_execution_timeout`` setting.\n\n The default is 60 seconds. Setting it to ``'0'`` disables the mechanism. Setting the timeout to less than ``2`` seconds is not recommended.\n\n Note that the actual time an idle connection can live can be up to two times longer than the specified timeout.\n\n This is a system-level config setting.\n\n:eql:synopsis:`session_idle_transaction_timeout -> std::duration`\n Sets the timeout for how long client connections can stay inactive while in a transaction.\n\n The default is 10 seconds. Setting it to ``'0'`` disables the mechanism.\n\n .. note::\n\n For ``session_idle_transaction_timeout`` and ``query_execution_timeout``, values under 1ms are rounded down to zero, which will disable the timeout. In order to set a timeout, please set a duration of 1ms or greater.\n\n ``session_idle_timeout`` can take values below 1ms.\n\n:eql:synopsis:`query_execution_timeout -> std::duration`\n Sets a time limit on how long a query can be run.\n\n Setting it to ``'0'`` disables the mechanism. The timeout isn't enabled by default.\n\n .. note::\n\n For ``session_idle_transaction_timeout`` and ``query_execution_timeout``, values under 1ms are rounded down to zero, which will disable the timeout. In order to set a timeout, please set a duration of 1ms or greater.\n\n ``session_idle_timeout`` can take values below 1ms.\n\n.. _ref_reference_environment:\n.. _ref_reference_envvar_variants:\n\nEnvironment variables\n=====================\n\nCertain behaviors of the Gel server are configured at startup. This configuration can be set with environment variables. The variables documented on this page are supported when using the |gel-server| binary or the official :ref:`Docker image `.\n\nSome environment variables (noted below) support ``_FILE`` and ``_ENV`` variants.\n\n- The ``_FILE`` variant expects its value to be a file name. The file's contents will be read and used as the value.\n- The ``_ENV`` variant expects its value to be the name of another environment variable. The value of the other environment variable is then used as the final value. This is convenient in deployment scenarios where relevant values are auto populated into fixed environment variables.\n\n.. note::\n\n For |Gel| versions before 6.0 the prefix for all environment variables is ``EDGEDB_`` instead of ``GEL_``.\n\nGEL_DEBUG_HTTP_INJECT_CORS\n--------------------------\n\nSet to ``1`` to have Gel send appropriate CORS headers with HTTP responses.\n\n.. note::\n\n This is set to ``1`` by default for Gel Cloud instances.\n\n\n.. _ref_reference_envvar_admin_ui:\n\nGEL_SERVER_ADMIN_UI\n-------------------\n\nSet to ``enabled`` to enable the web-based admininstrative UI for the instance.\n\nMaps directly to the |gel-server| flag ``--admin-ui``.\n\n\nGEL_SERVER_ALLOW_INSECURE_BINARY_CLIENTS\n----------------------------------------\n\n.. warning:: Deprecated\n\n Use :gelenv:`SERVER_BINARY_ENDPOINT_SECURITY` instead.\n\nSpecifies the security mode of the server's binary endpoint. When set to ``1``,\nnon-TLS connections are allowed. Not set by default.\n\n.. warning::\n\n Disabling TLS is not recommended in production.\n\n\nGEL_SERVER_ALLOW_INSECURE_HTTP_CLIENTS\n--------------------------------------\n\n.. warning:: Deprecated\n\n Use :gelenv:`SERVER_HTTP_ENDPOINT_SECURITY` instead.\n\nSpecifies the security mode of the server's HTTP endpoint. When set to ``1``,\nnon-TLS connections are allowed. Not set by default.\n\n.. warning::\n\n Disabling TLS is not recommended in production.\n\n\n.. _ref_reference_docker_gel_server_backend_dsn:\n\nGEL_SERVER_BACKEND_DSN / _FILE / _ENV\n-------------------------------------\n\nSpecifies a PostgreSQL connection string in the `URI format`_. If set, the\nPostgreSQL cluster specified by the URI is used instead of the builtin\nPostgreSQL server. Cannot be specified alongside :gelenv:`SERVER_DATADIR`. Maps directly to the |gel-server| flag ``--backend-dsn``.\n\nThe ``_FILE`` and ``_ENV`` variants are also supported.\n\n.. _URI format:\n https://www.postgresql.org/docs/13/libpq-connect.html#id-1.7.3.8.3.6\n\nGEL_SERVER_MAX_BACKEND_CONNECTIONS\n----------------------------------\n\nThe maximum NUM of connections this Gel instance could make to the backend\nPostgreSQL cluster. If not set, Gel will detect and calculate the NUM:\nRAM/100MiB for local Postgres, or pg_settings.max_connections for remote\nPostgres minus the NUM of ``--reserved-pg-connections``.\n\nGEL_SERVER_BINARY_ENDPOINT_SECURITY\n-----------------------------------\n\nSpecifies the security mode of the server's binary endpoint. When set to\n``optional``, non-TLS connections are allowed. Default is ``tls``.\n\n.. warning::\n\n Disabling TLS is not recommended in production.\n\n\nGEL_SERVER_BIND_ADDRESS / _FILE / _ENV\n--------------------------------------\n\nSpecifies the network interface on which Gel will listen. Maps directly to the |gel-server| flag ``--bind-address``.\n\nThe ``_FILE`` and ``_ENV`` variants are also supported.\n\n\nGEL_SERVER_BOOTSTRAP_COMMAND\n----------------------------\n\nUseful to fine-tune initial user creation and other initial setup. Maps directly to the |gel-server| flag ``--bootstrap-command``.\n\nThe ``_FILE`` and ``_ENV`` variants are also supported.\n\n.. note::\n\n A create branch statement (i.e., :eql:stmt:`create empty branch`, :eql:stmt:`create schema branch`, or :eql:stmt:`create data branch`) cannot be combined in a block with any other statements. Since all statements in :gelenv:`SERVER_BOOTSTRAP_COMMAND` run in a single block, it cannot be used to create a branch and, for example, create a user on that branch.\n\n For Docker deployments, you can instead write :ref:`custom scripts to run before migrations `. These are placed in ``/gel-bootstrap.d/``. By writing your ``create branch`` statements in one ``.edgeql`` file each placed in ``/gel-bootstrap.d/`` and other statements in their own file, you can create branches and still run other EdgeQL statements to bootstrap your instance.\n\n Note that for |EdgeDB| versions prior to 5.0, paths contain \"edgedb\" instead of \"gel\", so ``/gel-bootstrap.d/`` becomes ``/edgedb-bootstrap.d/``.\n\nGEL_SERVER_BOOTSTRAP_ONLY\n-------------------------\n\nWhen set, bootstrap the database cluster and exit. Not set by default.\n\n\n.. _ref_reference_docker_gel_server_datadir:\n\nGEL_SERVER_DATADIR\n------------------\n\nSpecifies a path where the database files are located. Default is\n``/var/lib/gel/data``. Cannot be specified alongside\n:gelenv:`SERVER_BACKEND_DSN`.\n\nMaps directly to the |gel-server| flag ``--data-dir``.\n\n\nGEL_SERVER_DEFAULT_AUTH_METHOD / _FILE / _ENV\n---------------------------------------------\n\nOptionally specifies the authentication method used by the server instance. Supported values are ``SCRAM`` (the default) and ``Trust``. When set to ``Trust``, the database will allow complete unauthenticated access for all who have access to the database port.\n\nThis is often useful when setting an admin password on an instance that lacks one.\n\nUse at your own risk and only for development and testing.\n\nThe ``_FILE`` and ``_ENV`` variants are also supported.\n\n\nGEL_SERVER_HTTP_ENDPOINT_SECURITY\n---------------------------------\n\nSpecifies the security mode of the server's HTTP endpoint. When set to ``optional``, non-TLS connections are allowed. Default is ``tls``.\n\n.. warning::\n\n Disabling TLS is not recommended in production.\n\n\nGEL_SERVER_INSTANCE_NAME\n------------------------\n\nSpecify the server instance name.\n\n\nGEL_SERVER_JWS_KEY_FILE\n-----------------------\n\nSpecifies a path to a file containing a public key in PEM format used to verify JWT signatures. The file could also contain a private key to sign JWT for local testing.\n\n\nGEL_SERVER_LOG_LEVEL\n--------------------\n\nSet the logging level. Default is ``info``. Other possible values are ``debug``, ``warn``, ``error``, and ``silent``.\n\n\nGEL_SERVER_PORT / _FILE / _ENV\n------------------------------\n\nSpecifies the network port on which Gel will listen. Default is ``5656``. Maps directly to the |gel-server| flag ``--port``.\n\nThe ``_FILE`` and ``_ENV`` variants are also supported.\n\n\nGEL_SERVER_RUNSTATE_DIR\n-----------------------\n\nSpecifies a path where Gel will place its Unix socket and other transient files. Maps directly to the |gel-server| flag ``--runstate-dir``.\n\n\nGEL_SERVER_SECURITY\n-------------------\n\nWhen set to ``insecure_dev_mode``, sets :gelenv:`SERVER_DEFAULT_AUTH_METHOD` to ``Trust``, and :gelenv:`SERVER_TLS_CERT_MODE` to ``generate_self_signed`` (unless an explicit TLS certificate is specified). Finally, if this option is set, the server will accept plaintext HTTP connections. Maps directly to the |gel-server| flag ``--security``.\n\n.. warning::\n\n Disabling TLS is not recommended in production.\n\n\n\nGEL_SERVER_TLS_CERT_FILE\n------------------------\n\nThe TLS certificate file, exclusive with :gelenv:`SERVER_TLS_CERT_MODE=generate_self_signed`. Maps directly to the |gel-server| flag ``--tls-cert-file``.\n\nGEL_SERVER_TLS_KEY_FILE\n-----------------------\n\nThe TLS private key file, exclusive with :gelenv:`SERVER_TLS_CERT_MODE=generate_self_signed`. Maps directly to the |gel-server| flag ``--tls-key-file``.\n\n\nGEL_SERVER_TLS_CERT_MODE / _FILE / _ENV\n---------------------------------------\n\nSpecifies what to do when the TLS certificate and key are either not specified or are missing.\n\n- When set to ``require_file``, the TLS certificate and key must be specified in the :gelenv:`SERVER_TLS_CERT` and :gelenv:`SERVER_TLS_KEY` variables and both must exist.\n- When set to ``generate_self_signed`` a new self-signed certificate and private key will be generated and placed in the path specified by :gelenv:`SERVER_TLS_CERT` and :gelenv:`SERVER_TLS_KEY`, if those are set. Otherwise, the generated certificate and key are stored as ``edbtlscert.pem`` and ``edbprivkey.pem`` in :gelenv:`SERVER_DATADIR`, or, if :gelenv:`SERVER_DATADIR` is not set, they will be placed in ``/etc/ssl/gel``.\n\nDefault is ``generate_self_signed`` when :gelenv:`SERVER_SECURITY=insecure_dev_mode`. Otherwise, the default is ``require_file``.\n\nMaps directly to the |gel-server| flag ``--tls-cert-mode``.\n\nThe ``_FILE`` and ``_ENV`` variants are also supported.\n\nDocker image specific variables\n===============================\n\nThese variables are only used by the Docker image. Setting these variables outside that context will have no effect.\n\n\nGEL_DOCKER_ABORT_CODE\n---------------------\n\nIf the process fails, the arguments are logged to stderr and the script is terminated with this exit code. Default is ``1``.\n\n\nGEL_DOCKER_APPLY_MIGRATIONS\n---------------------------\n\nThe container will attempt to apply migrations in ``dbschema/migrations`` unless this variable is set to ``never``.\n\n**Values**: ``always`` (default), ``never``\n\n\nGEL_DOCKER_BOOTSTRAP_TIMEOUT_SEC\n--------------------------------\n\nSets the number of seconds to wait for instance bootstrapping to complete before timing out. Default is ``300``.\n\n\nGEL_DOCKER_LOG_LEVEL\n--------------------\n\nChange the logging level for the docker container.\n\n**Values**: ``trace``, ``debug``, ``info`` (default), ``warn``, ``error``\n\n\nGEL_DOCKER_SHOW_GENERATED_CERT\n------------------------------\n\nShows the generated TLS certificate in console output.\n\n**Values**: ``always`` (default), ``never``\n\n\nGEL_SERVER_BINARY\n-----------------\n\nSets the Gel server binary to run. Default is |gel-server|.\n\n\nGEL_SERVER_BOOTSTRAP_COMMAND_FILE\n---------------------------------\n\nRun the script when initializing the database. The script is run by the default user within the default |branch|. May be used with or without :gelenv:`SERVER_BOOTSTRAP_ONLY`.\n\n\nGEL_SERVER_COMPILER_POOL_MODE\n-----------------------------\n\nChoose a mode for the compiler pool to scale. ``fixed`` means the pool will not scale and sticks to :gelenv:`SERVER_COMPILER_POOL_SIZE`, while ``on_demand`` means the pool will maintain at least 1 worker and automatically scale up (to :gelenv:`SERVER_COMPILER_POOL_SIZE` workers ) and down to the demand.\n\n**Values**: ``fixed``, ``on_demand``\n\nDefault is ``fixed`` in production mode and ``on_demand`` in development mode.\n\n\nGEL_SERVER_COMPILER_POOL_SIZE\n-----------------------------\n\nWhen :gelenv:`SERVER_COMPILER_POOL_MODE` is ``fixed``, this setting is the exact size of the compiler pool. When :gelenv:`SERVER_COMPILER_POOL_MODE` is ``on_demand``, this will serve as the maximum size of the compiler pool.\n\n\nGEL_SERVER_EMIT_SERVER_STATUS\n-----------------------------\n\nInstruct the server to emit changes in status to *DEST*, where *DEST* is a URI specifying a file (``file://``), or a file descriptor (``fd://``). If the URI scheme is not specified, ``file://`` is assumed.\n\n\nGEL_SERVER_EXTRA_ARGS\n---------------------\n\nAdditional arguments to pass when starting the Gel server.\n\n\nGEL_SERVER_PASSWORD / _FILE / _ENV\n----------------------------------\n\nThe password for the default superuser account (or the user specified in :gelenv:`SERVER_USER`) will be set to this value. If no value is provided, a password will not be set, unless set via :gelenv:`SERVER_BOOTSTRAP_COMMAND`. (If a value for :gelenv:`SERVER_BOOTSTRAP_COMMAND` is provided, this variable will be ignored.)\n\nThe ``_FILE`` and ``_ENV`` variants are also supported.\n\n\nGEL_SERVER_PASSWORD_HASH / _FILE / _ENV\n---------------------------------------\n\nA variant of :gelenv:`SERVER_PASSWORD`, where the specified value is a hashed password verifier instead of plain text.\n\nIf :gelenv:`SERVER_BOOTSTRAP_COMMAND` is set, this variable will be ignored.\n\nThe ``_FILE`` and ``_ENV`` variants are also supported.\n\n\nGEL_SERVER_TENANT_ID\n--------------------\n\nSpecifies the tenant ID of this server. When using multiple Gel instances with one Postgres cluster each Gel instance must have a unique tenant ID. Must be an alphanumeric ASCII string, maximum 10 characters long. Defaults to \"E\" if not set.\n\n\nGEL_SERVER_UID\n--------------\n\nSpecifies the ID of the user which should run the server binary. Default is ``1``.\n\n\nGEL_SERVER_USER\n---------------\n\nIf set to anything other than the default username |admin|, the username specified will be created. The user defined here will be the one assigned the password set in :gelenv:`SERVER_PASSWORD` or the hash set in :gelenv:`SERVER_PASSWORD_HASH`.\n" }, { "path": "docs/reference/running/deployment/aws_aurora_ecs.rst", "content": ".. _ref_guide_deployment_aws_aurora_ecs:\n\n===\nAWS\n===\n\n:edb-alt-title: Deploying Gel to AWS\n\n.. note::\n\n We recomend using our `helm chart `_ to deploy gel on AWS EKS. The\n CloudFormation guide below does not configure TLS certificates correctly.\n\n.. _helm-chart:\n https://github.com/geldata/helm-charts/blob/main\n /charts/gel-server/README.md\n\nIn this guide we show how to deploy Gel on AWS using Amazon Aurora and\nElastic Container Service.\n\n.. include:: ./note_cloud_reset_password.rst\n\nPrerequisites\n=============\n\n* AWS account with billing enabled (or a `free trial `_)\n* (optional) ``aws`` CLI (`install `_)\n\n.. _aws-trial: https://aws.amazon.com/free\n.. _awscli-install:\n https://docs.aws.amazon.com\n /cli/latest/userguide/getting-started-install.html\n\nQuick Install with CloudFormation\n=================================\n\nWe maintain a `CloudFormation template `_ for easy automated\ndeployment of Gel in your AWS account. The template deploys Gel\nto a new ECS service and connects it to a newly provisioned Aurora PostgreSQL\ncluster. The created instance has a public IP address with TLS configured and\nis protected by a password you provide.\n\nCloudFormation Web Portal\n-------------------------\n\nClick `here `_ to start the deployment process using CloudFormation\nportal and follow the prompts. You'll be prompted to provide a value for the\nfollowing parameters:\n\n- ``DockerImage``: defaults to the latest version (``geldata/gel``), or you\n can specify a particular tag from the ones published to `Docker Hub\n `_.\n- ``InstanceName``: āš ļø Due to limitations with AWS, this must be 22 characters\n or less!\n- ``SuperUserPassword``: this will be used as the password for the new Gel\n instance. Keep track of the value you provide.\n\nOnce the deployment is complete, follow these steps to find the host name that\nhas been assigned to your Gel instance:\n\n.. lint-off\n\n1. Open the AWS Console and navigate to CloudFormation > Stacks. Click on the\n newly created Stack.\n2. Wait for the status to read ``CREATE_COMPLETE``ā€”it can take 15 minutes or\n more.\n3. Once deployment is complete, click the ``Outputs`` tab. The value of\n ``PublicHostname`` is the hostname at which your Gel instance is\n publicly available.\n4. Copy the hostname and run the following command to open a REPL to your\n instance.\n\n .. code-block:: bash\n\n $ gel --dsn gel://admin:@ --tls-security insecure\n Gel x.x\n Type \\help for help, \\quit to quit.\n gel>\n\n.. lint-on\n\nTo make changes to your Gel deployment like upgrading the Gel version or\nenabling the UI you can follow the CloudFormation\n`Updating a stack `_ instructions. Search for\n``ContainerDefinitions`` in the template and you will find where Gel's\n:ref:`environment variables ` are\ndefined. To upgrade the Gel version specify a\n`docker image tag `_ with the image name ``geldata/gel`` in the\nsecond step of the update workflow.\n\nCloudFormation CLI\n------------------\n\nAlternatively, if you prefer to use AWS CLI, run the following command in\nyour terminal:\n\n.. code-block:: bash\n\n $ aws cloudformation create-stack \\\n --stack-name Gel \\\n --template-url \\\n https://gel-deployment.s3.us-east-2.amazonaws.com/gel-aurora.yml \\\n --capabilities CAPABILITY_NAMED_IAM \\\n --parameters ParameterKey=SuperUserPassword,ParameterValue=\n\n\n.. _cf-template: https://github.com/geldata/gel-deploy/tree/dev/aws-cf\n.. _cf-deploy:\n https://console.aws.amazon.com\n /cloudformation/home#/stacks/new?stackName=Gel&templateURL=\n https%3A%2F%2Fgel-deployment.s3.us-east-2.amazonaws.com%2Fgel-aurora.yml\n.. _aws_console:\n https://console.aws.amazon.com\n /ec2/v2/home#NIC:search=ec2-security-group\n.. _stack-update:\n https://docs.aws.amazon.com\n /AWSCloudFormation/latest/UserGuide/cfn-whatis-howdoesitwork.html\n.. _docker-tags: https://hub.docker.com/r/geldata/gel/tags\n\n\nConnecting your application\n===========================\n\nTo connect your application to the Gel instance, you'll need to provide\nconnection parameters. Gel client libraries can be configured using either\na DSN (connection string) or individual environment variables.\n\nObtaining connection parameters\n-------------------------------\n\nYour connection requires the following components:\n\n- **Host**: The ``PublicHostname`` value from the CloudFormation Stack's\n ``Outputs`` tab.\n- **Port**: ``5656`` (the default Gel port)\n- **Username**: |admin| (the default superuser)\n- **Password**: The ``SuperUserPassword`` you specified during deployment\n- **Branch**: |main| (the default branch)\n\nConstruct the DSN using these values:\n\n.. code-block:: bash\n\n $ GEL_DSN=\"gel://admin:@:5656\"\n\nObtaining the TLS certificate\n-----------------------------\n\n.. warning::\n\n The CloudFormation template does not configure TLS certificates correctly.\n We recommend using ``--tls-security insecure`` for testing, but for\n production you should use our `helm chart `_ or configure\n TLS manually.\n\nTo connect securely, your application needs the server's TLS certificate.\nFor self-signed certificates, you can retrieve the certificate by connecting\nto the instance and extracting it:\n\n.. code-block:: bash\n\n $ gel --dsn $GEL_DSN --tls-security insecure \\\n query \"SELECT sys::get_tls_certificate()\"\n\nStore this certificate and provide it to your application via the\n:gelenv:`TLS_CA` or :gelenv:`TLS_CA_FILE` environment variable.\n\nUsing in your application\n-------------------------\n\nSet these environment variables where you deploy your application:\n\n.. code-block:: bash\n\n GEL_DSN=\"gel://admin:@:5656\"\n # For self-signed certificates:\n GEL_CLIENT_TLS_SECURITY=insecure\n # Or with a proper TLS certificate:\n GEL_TLS_CA=\"\"\n\nGel's client libraries will automatically read these environment variables.\n\nLocal development with the CLI\n------------------------------\n\nTo make your remote instance easier to work with during local development,\ncreate an alias using :gelcmd:`instance link`.\n\n.. note::\n\n The command groups :gelcmd:`instance` and :gelcmd:`project` are not\n intended to manage production instances.\n\n.. code-block:: bash\n\n $ gel instance link \\\n --dsn $GEL_DSN \\\n --non-interactive \\\n --trust-tls-cert \\\n my_aws_instance\n\nYou can now refer to the remote instance using the alias ``my_aws_instance``.\nUse this alias wherever an instance name is expected:\n\n.. code-block:: bash\n\n $ gel -I my_aws_instance\n Gel x.x\n Type \\help for help, \\quit to quit.\n gel>\n\nOr apply migrations:\n\n.. code-block:: bash\n\n $ gel -I my_aws_instance migrate\n\n\nHealth Checks\n=============\n\nUsing an HTTP client, you can perform health checks to monitor the status of\nyour Gel instance. Learn how to use them with our :ref:`health checks guide\n`.\n" }, { "path": "docs/reference/running/deployment/azure_flexibleserver.rst", "content": ".. _ref_guide_deployment_azure_flexibleserver:\n\n=====\nAzure\n=====\n\n:edb-alt-title: Deploying Gel to Azure\n\nIn this guide we show how to deploy Gel using Azure's `Postgres\nFlexible Server\n`_ as the\nbackend.\n\n.. include:: ./note_cloud_reset_password.rst\n\nPrerequisites\n=============\n\n* Valid Azure Subscription with billing enabled or credits (`free trial\n `_).\n* Azure CLI (`install `_).\n\n.. _azure-trial: https://azure.microsoft.com/en-us/free/\n.. _azure-install: https://docs.microsoft.com/en-us/cli/azure/install-azure-cli\n\n\nProvision a Gel instance\n=========================\n\nLogin to your Microsoft Azure account.\n\n.. code-block:: bash\n\n $ az login\n\nCreate a new resource group.\n\n.. code-block:: bash\n\n $ GROUP=my-group-name\n $ az group create --name $GROUP --location westus\n\nProvision a PostgreSQL server.\n\n.. note::\n\n If you already have a database provisioned you can skip this step.\n\nFor convenience, assign a value to the ``PG_SERVER_NAME`` environment\nvariable; we'll use this variable in multiple later commands.\n\n.. code-block:: bash\n\n $ PG_SERVER_NAME=postgres-for-gel\n\nUse the ``read`` command to securely assign a value to the ``PASSWORD``\nenvironment variable.\n\n.. code-block:: bash\n\n $ echo -n \"> \" && read -s PASSWORD\n\nThen create a Postgres Flexible server.\n\n.. code-block:: bash\n\n $ az postgres flexible-server create \\\n --resource-group $GROUP \\\n --name $PG_SERVER_NAME \\\n --location westus \\\n --admin-user gel_admin \\\n --admin-password $PASSWORD \\\n --sku-name Standard_D2s_v3 \\\n --version 14 \\\n --yes\n\n.. note::\n\n If you get an error saying ``\"Specified server name is already used.\"``\n change the value of ``PG_SERVER_NAME`` and rerun the command.\n\nAllow other Azure services access to the Postgres instance.\n\n.. code-block:: bash\n\n $ az postgres flexible-server firewall-rule create \\\n --resource-group $GROUP \\\n --name $PG_SERVER_NAME \\\n --rule-name allow-azure-internal \\\n --start-ip-address 0.0.0.0 \\\n --end-ip-address 0.0.0.0\n\n|Gel| requires Postgres' ``uuid-ossp`` extension which needs to be enabled.\n\n.. code-block:: bash\n\n $ az postgres flexible-server parameter set \\\n --resource-group $GROUP \\\n --server-name $PG_SERVER_NAME \\\n --name azure.extensions \\\n --value uuid-ossp\n\nAzure is not able to reliably pull docker images `because of rate limits\n`_, so you will need to provide docker hub login credentials\nto create a container. If you don't already have a docker hub account you can\ncreate one `here `_.\n\n.. _azure-cli-issue: https://github.com/Azure/azure-cli/issues/29300\n\n.. code-block:: bash\n\n $ echo -n \"docker user> \" && read -s DOCKER_USER\n $ echo -n \"docker password> \" && read -s DOCKER_PASSWORD\n\nStart a Gel container.\n\n.. code-block:: bash\n\n $ PG_HOST=$(\n az postgres flexible-server list \\\n --resource-group $GROUP \\\n --query \"[?name=='$PG_SERVER_NAME'].fullyQualifiedDomainName | [0]\" \\\n --output tsv\n )\n $ DSN=\"postgresql://gel_admin:$PASSWORD@$PG_HOST/postgres?sslmode=require\"\n $ az container create \\\n --registry-username $DOCKER_USER \\\n --registry-password $DOCKER_PASSWORD \\\n --registry-login-server index.docker.io \\\n --os-type Linux \\\n --cpu 1 \\\n --memory 1 \\\n --resource-group $GROUP \\\n --name gel-container-group \\\n --image geldata/gel \\\n --dns-name-label geldb \\\n --ports 5656 \\\n --secure-environment-variables \\\n \"GEL_SERVER_PASSWORD=$PASSWORD\" \\\n \"GEL_SERVER_BACKEND_DSN=$DSN\" \\\n --environment-variables \\\n GEL_SERVER_TLS_CERT_MODE=generate_self_signed\n\nPersist the SSL certificate. We have configured Gel to generate a self\nsigned SSL certificate when it starts. However, if the container is restarted a\nnew certificate would be generated. To preserve the certificate across failures\nor reboots copy the certificate files and use their contents in the\n:gelenv:`SERVER_TLS_KEY` and :gelenv:`SERVER_TLS_CERT` environment variables.\n\n.. code-block:: bash\n\n $ key=\"$( az container exec \\\n --resource-group $GROUP \\\n --name gel-container-group \\\n --exec-command \"cat /tmp/gel/edbprivkey.pem\" \\\n | tr -d \"\\r\" )\"\n $ cert=\"$( az container exec \\\n --resource-group $GROUP \\\n --name gel-container-group \\\n --exec-command \"cat /tmp/gel/edbtlscert.pem\" \\\n | tr -d \"\\r\" )\"\n $ az container delete \\\n --resource-group $GROUP \\\n --name gel-container-group \\\n --yes\n $ az container create \\\n --registry-username $DOCKER_USER \\\n --registry-password $DOCKER_PASSWORD \\\n --registry-login-server index.docker.io \\\n --os-type Linux \\\n --cpu 1 \\\n --memory 1 \\\n --resource-group $GROUP \\\n --name gel-container-group \\\n --image geldata/gel \\\n --dns-name-label geldb \\\n --ports 5656 \\\n --secure-environment-variables \\\n \"GEL_SERVER_PASSWORD=$PASSWORD\" \\\n \"GEL_SERVER_BACKEND_DSN=$DSN\" \\\n \"GEL_SERVER_TLS_KEY=$key\" \\\n --environment-variables \\\n \"GEL_SERVER_TLS_CERT=$cert\"\n\n\nConnecting your application\n===========================\n\nTo connect your application to the Gel instance, you'll need to provide\nconnection parameters. Gel client libraries can be configured using either\na DSN (connection string) or individual environment variables.\n\nObtaining connection parameters\n-------------------------------\n\nYour connection requires the following components:\n\n- **Host**: The FQDN of your Azure container instance. Retrieve it with:\n\n .. code-block:: bash\n\n $ az container list \\\n --resource-group $GROUP \\\n --query \"[?name=='gel-container-group'].ipAddress.fqdn | [0]\" \\\n --output tsv\n\n- **Port**: ``5656`` (the default Gel port)\n- **Username**: |admin| (the default superuser)\n- **Password**: The password you set in the ``$PASSWORD`` variable\n- **Branch**: |main| (the default branch)\n\nConstruct the DSN using these values:\n\n.. code-block:: bash\n\n $ GEL_HOST=$(az container list \\\n --resource-group $GROUP \\\n --query \"[?name=='gel-container-group'].ipAddress.fqdn | [0]\" \\\n --output tsv)\n $ GEL_DSN=\"gel://admin:$PASSWORD@$GEL_HOST:5656\"\n\nObtaining the TLS certificate\n-----------------------------\n\nSince we configured Gel with a self-signed TLS certificate, your application\nneeds the certificate to connect securely. Retrieve it from the container:\n\n.. code-block:: bash\n\n $ az container exec \\\n --resource-group $GROUP \\\n --name gel-container-group \\\n --exec-command \"cat /tmp/gel/edbtlscert.pem\" \\\n | tr -d \"\\r\" > gel-tls-cert.pem\n\nAlternatively, you can retrieve it using the Gel CLI:\n\n.. code-block:: bash\n\n $ gel --dsn $GEL_DSN --tls-security insecure \\\n query \"SELECT sys::get_tls_certificate()\" > gel-tls-cert.pem\n\nUsing in your application\n-------------------------\n\nSet these environment variables where you deploy your application:\n\n.. code-block:: bash\n\n GEL_DSN=\"gel://admin:@:5656\"\n # For self-signed certificates, either trust the cert:\n GEL_TLS_CA_FILE=\"/path/to/gel-tls-cert.pem\"\n # Or (for development only) disable TLS verification:\n GEL_CLIENT_TLS_SECURITY=insecure\n\nGel's client libraries will automatically read these environment variables.\n\nLocal development with the CLI\n------------------------------\n\nTo make your remote instance easier to work with during local development,\ncreate an alias using :gelcmd:`instance link`.\n\n.. note::\n\n The command groups :gelcmd:`instance` and :gelcmd:`project` are not\n intended to manage production instances.\n\n.. code-block:: bash\n\n $ printf $PASSWORD | gel instance link \\\n --dsn $GEL_DSN \\\n --password-from-stdin \\\n --non-interactive \\\n --trust-tls-cert \\\n my_azure_instance\n\nYou can now refer to the remote instance using the alias ``my_azure_instance``.\nUse this alias wherever an instance name is expected:\n\n.. code-block:: bash\n\n $ gel -I my_azure_instance\n Gel x.x\n Type \\help for help, \\quit to quit.\n gel>\n\nOr apply migrations:\n\n.. code-block:: bash\n\n $ gel -I my_azure_instance migrate\n\n\nHealth Checks\n=============\n\nUsing an HTTP client, you can perform health checks to monitor the status of\nyour Gel instance. Learn how to use them with our :ref:`health checks guide\n`.\n" }, { "path": "docs/reference/running/deployment/bare_metal.rst", "content": ".. _ref_guide_deployment_bare_metal:\n\n==========\nBare Metal\n==========\n\n:edb-alt-title: Deploying Gel to a Bare Metal Server\n\nIn this guide we show how to deploy Gel to bare metal using your system's\npackage manager and systemd.\n\n.. include:: ./note_cloud_reset_password.rst\n\nInstall the Gel Package\n=======================\n\nThe steps for installing the Gel package will be slightly different\ndepending on your Linux distribution. Once you have the package installed you\ncan jump to :ref:`ref_guide_deployment_bare_metal_enable_unit`.\n\n\nDebian/Ubuntu LTS\n-----------------\nImport the Gel packaging key.\n\n.. code-block:: bash\n\n $ sudo mkdir -p /usr/local/share/keyrings && \\\n sudo curl --proto '=https' --tlsv1.2 -sSf \\\n -o /usr/local/share/keyrings/gel-keyring.gpg \\\n https://packages.geldata.com/keys/gel-keyring.gpg\n\nAdd the Gel package repository.\n\n.. code-block:: bash\n\n $ echo deb '[signed-by=/usr/local/share/keyrings/gel-keyring.gpg]' \\\n https://packages.geldata.com/apt \\\n $(grep \"VERSION_CODENAME=\" /etc/os-release | cut -d= -f2) main \\\n | sudo tee /etc/apt/sources.list.d/gel.list\n\n.. note::\n\n For non-LTS releases of Debian/Ubuntu (e.g. Ubuntu Oracular), one can install\n package for latest LTS release, because they are usually forward compatible.\n To do this, replace the ``$(grep ...)`` with the name of latest LTS release\n (e.g. ``noble``).\n\nInstall the Gel package.\n\n.. code-block:: bash\n\n $ sudo apt-get update && sudo apt-get install gel-6\n\n\nCentOS/RHEL 7/8\n---------------\nAdd the Gel package repository.\n\n.. code-block:: bash\n\n $ sudo curl --proto '=https' --tlsv1.2 -sSfL \\\n https://packages.geldata.com/rpm/gel-rhel.repo \\\n > /etc/yum.repos.d/gel.repo\n\nInstall the Gel package.\n\n.. code-block:: bash\n\n $ sudo yum install gel-6\n\nDisable SELinux.\n\n.. code-block:: bash\n\n $ sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config\n $ reboot\n\n\n.. _ref_guide_deployment_bare_metal_enable_unit:\n\nEnable a systemd unit\n=====================\n\nThe Gel package comes bundled with a systemd unit that is disabled by\ndefault. You can start the server by enabling the unit.\n\n.. code-block:: bash\n\n $ sudo systemctl enable --now gel-server-6\n\nThis will start the server on port 5656, and the data directory will be\n``/var/lib/gel/6/data``.\n\n.. warning::\n\n |gel-server| cannot be run as root.\n\nSet environment variables\n=========================\n\nTo set environment variables when running Gel with ``systemctl``,\n\n.. code-block:: bash\n\n $ systemctl edit --full gel-server-6\n\nThis opens a ``systemd`` unit file. Set the desired environment variables\nunder the ``[Service]`` section. View the supported environment variables at\n:ref:`Reference > Environment Variables `.\n\n.. code-block:: toml\n\n [Service]\n Environment=\"GEL_SERVER_TLS_CERT_MODE=generate_self_signed\"\n Environment=\"GEL_SERVER_ADMIN_UI=enabled\"\n\nSave the file and exit, then restart the service.\n\n.. code-block:: bash\n\n $ systemctl restart gel-server-6\n\n\nSet a password\n==============\nThere is no default password. To set one, you will first need to get the Unix\nsocket directory. You can find this by looking at your system.d unit file.\n\n.. code-block:: bash\n\n $ sudo systemctl cat gel-server-6\n\nSet a password by connecting from localhost.\n\n.. code-block:: bash\n\n $ echo -n \"> \" && read -s PASSWORD\n $ RUNSTATE_DIR=$(systemctl show gel-server-6 -P ExecStart | \\\n grep -o -m 1 -- \"--runstate-dir=[^ ]\\+\" | \\\n awk -F \"=\" '{print $2}')\n $ sudo gel --port 5656 --tls-security insecure --admin \\\n --unix-path $RUNSTATE_DIR \\\n query \"ALTER ROLE admin SET password := '$PASSWORD'\"\n\nThe server listens on localhost by default. Changing this looks like this.\n\n.. code-block:: bash\n\n $ gel --port 5656 --tls-security insecure --password query \\\n \"CONFIGURE INSTANCE SET listen_addresses := {'0.0.0.0'};\"\n\nThe listen port can be changed from the default ``5656`` if your deployment\nscenario requires a different value.\n\n.. code-block:: bash\n\n $ gel --port 5656 --tls-security insecure --password query \\\n \"CONFIGURE INSTANCE SET listen_port := 1234;\"\n\nYou may need to restart the server after changing the listen port or addresses.\n\n.. code-block:: bash\n\n $ sudo systemctl restart gel-server-6\n\n\nConnecting your application\n===========================\n\nTo connect your application to the Gel instance, you'll need to provide\nconnection parameters. Gel client libraries can be configured using either\na DSN (connection string) or individual environment variables.\n\nObtaining connection parameters\n-------------------------------\n\nYour connection requires the following components:\n\n- **Host**: The IP address or hostname of your server (e.g., ``localhost``,\n ``192.168.1.100``, or ``gel.example.com``)\n- **Port**: ``5656`` by default, or the custom port if you changed it with\n ``CONFIGURE INSTANCE SET listen_port``\n- **Username**: |admin| (the default superuser)\n- **Password**: The password you set with ``ALTER ROLE admin SET password``\n- **Branch**: |main| (the default branch)\n\nConstruct the DSN using these values:\n\n.. code-block:: bash\n\n $ GEL_DSN=\"gel://admin:@:5656\"\n\nObtaining the TLS certificate\n-----------------------------\n\nIf you configured Gel with ``GEL_SERVER_TLS_CERT_MODE=generate_self_signed``,\nyour application needs the certificate to connect securely.\n\nThe generated certificate is stored in the data directory. You can find it at:\n\n.. code-block:: bash\n\n $ cat /var/lib/gel/6/data/edbtlscert.pem\n\nAlternatively, retrieve it using the Gel CLI:\n\n.. code-block:: bash\n\n $ gel --dsn $GEL_DSN --tls-security insecure \\\n query \"SELECT sys::get_tls_certificate()\"\n\nUsing in your application\n-------------------------\n\nSet these environment variables where you deploy your application:\n\n.. code-block:: bash\n\n GEL_DSN=\"gel://admin:@:5656\"\n # For self-signed certificates, provide the CA cert:\n GEL_TLS_CA_FILE=\"/path/to/edbtlscert.pem\"\n # Or embed the certificate content directly:\n GEL_TLS_CA=\"\"\n\nGel's client libraries will automatically read these environment variables.\n\nLocal development with the CLI\n------------------------------\n\nTo make your instance easier to work with during local development,\ncreate an alias using :gelcmd:`instance link`.\n\n.. note::\n\n The command groups :gelcmd:`instance` and :gelcmd:`project` are not\n intended to manage production instances.\n\n.. code-block:: bash\n\n $ gel instance link \\\n --dsn $GEL_DSN \\\n --non-interactive \\\n --trust-tls-cert \\\n my_bare_metal_instance\n\nYou can now refer to the instance using the alias ``my_bare_metal_instance``.\nUse this alias wherever an instance name is expected:\n\n.. code-block:: bash\n\n $ gel -I my_bare_metal_instance\n Gel x.x\n Type \\help for help, \\quit to quit.\n gel>\n\nOr apply migrations:\n\n.. code-block:: bash\n\n $ gel -I my_bare_metal_instance migrate\n\n\nUpgrading Gel\n=============\n\nWhen you want to upgrade to the newest point release upgrade the package and\nrestart the ``gel-server-6`` unit.\n\n\nDebian/Ubuntu LTS\n-----------------\n\n.. code-block:: bash\n\n $ sudo apt-get update && sudo apt-get install --only-upgrade gel-6\n $ sudo systemctl restart gel-server-6\n\n\nCentOS/RHEL 7/8\n---------------\n\n.. code-block:: bash\n\n $ sudo yum update gel-6\n $ sudo systemctl restart gel-server-6\n\nHealth Checks\n=============\n\nUsing an HTTP client, you can perform health checks to monitor the status of\nyour Gel instance. Learn how to use them with our :ref:`health checks guide\n`.\n" }, { "path": "docs/reference/running/deployment/digitalocean.rst", "content": ".. _ref_guide_deployment_digitalocean:\n\n============\nDigitalOcean\n============\n\n:edb-alt-title: Deploying Gel to DigitalOcean\n\nCreate a droplet and use the :ref:`ref_guide_deployment_bare_metal` guide to\ninstall gel server.\n" }, { "path": "docs/reference/running/deployment/docker.rst", "content": ".. _ref_guide_deployment_docker:\n\n======\nDocker\n======\n\n:edb-alt-title: Deploying Gel with Docker\n\n.. include:: ./note_cloud_reset_password.rst\n\nWhen to use the \"geldata/gel\" Docker image\n==========================================\n\n.. _geldata/gel: https://hub.docker.com/r/geldata/gel\n\nThis image is primarily intended to be used directly when there is a\nrequirement to use Docker containers, such as in production, or in a\ndevelopment setup that involves multiple containers orchestrated by Docker\nCompose or a similar tool. Otherwise, using the :ref:`ref_cli_gel_server`\nCLI on the host system is the recommended way to install and run Gel\nservers.\n\n\nHow to use this image\n=====================\n\nThe simplest way to run the image (without data persistence) is this:\n\n.. code-block:: bash\n\n $ docker run --name gel -d \\\n -e GEL_SERVER_SECURITY=insecure_dev_mode \\\n geldata/gel\n\nSee the :ref:`ref_guides_deployment_docker_customization` section below for the\nmeaning of the :gelenv:`SERVER_SECURITY` variable and other options.\n\nThen, to authenticate to the Gel instance and store the credentials in a\nDocker volume, run:\n\n.. code-block:: bash\n\n $ docker run -it --rm --link=gel \\\n -e GEL_SERVER_PASSWORD=secret \\\n -v gel-cli-config:/.config/edgedb geldata/gel-cli \\\n -H gel instance link my_instance \\\n --tls-security insecure \\\n --non-interactive\n\nNow, to open an interactive shell to the database instance run this:\n\n.. code-block:: bash\n\n $ docker run -it --rm --link=gel \\\n -v gel-cli-config:/.config/edgedb geldata/gel-cli \\\n -I my_instance\n\n\nData Persistence\n================\n\nIf you want the contents of the database to survive container restarts, you\nmust mount a persistent volume at the path specified by\n:gelenv:`SERVER_DATADIR` (``/var/lib/gel/data`` by default). For example:\n\n.. code-block:: bash\n\n $ docker run \\\n --name gel \\\n -e GEL_SERVER_PASSWORD=secret \\\n -e GEL_SERVER_TLS_CERT_MODE=generate_self_signed \\\n -v /my/data/directory:/var/lib/gel/data \\\n -d geldata/gel\n\nNote that on Windows you must use a Docker volume instead:\n\n.. code-block:: bash\n\n $ docker volume create --name=gel-data\n $ docker run \\\n --name gel \\\n -e GEL_SERVER_PASSWORD=secret \\\n -e GEL_SERVER_TLS_CERT_MODE=generate_self_signed \\\n -v gel-data:/var/lib/gel/data \\\n -d geldata/gel\n\nIt is also possible to run a ``gel`` container on a remote PostgreSQL\ncluster specified by :gelenv:`SERVER_BACKEND_DSN`. See below for details.\n\n\nSchema Migrations\n=================\n\nA derived image may include application schema and migrations in ``/dbschema``,\nin which case the container will attempt to apply the schema migrations found\nin ``/dbschema/migrations``, unless the :gelenv:`DOCKER_APPLY_MIGRATIONS`\nenvironment variable is set to ``never``.\n\n\nDocker Compose\n==============\n\nA simple ``docker-compose`` configuration might look like this.\nWith a ``docker-compose.yaml`` containing:\n\n.. code-block:: yaml\n\n services:\n gel:\n image: geldata/gel\n environment:\n GEL_SERVER_SECURITY: insecure_dev_mode\n volumes:\n - \"./dbschema:/dbschema\"\n ports:\n - \"5656:5656\"\n\nOnce there is a :ref:`schema ` in ``dbschema/`` a\nmigration can be created with:\n\n.. code-block:: bash\n\n $ gel --tls-security=insecure -P 5656 migration create\n\nAlternatively, if you don't have the Gel CLI installed on your host\nmachine, you can use the CLI bundled with the server container:\n\n.. code-block:: bash\n\n $ docker compose exec gel \\\n gel --tls-security=insecure -P 5656 migration create\n\n\n.. _ref_guides_deployment_docker_customization:\n\nConfiguration\n=============\n\nThe Docker image supports the same set of enviroment variables as the Gel\nserver process, which are documented under :ref:`Reference > Environment\nVariables `.\n\n|Gel| containers can be additionally configured using initialization scripts\nand some Docker-specific environment variables, documented below.\n\n.. note::\n\n Some variables support ``_ENV`` and ``_FILE`` :ref:`variants\n ` to support more advanced configurations.\n\n.. _ref_guides_deployment_docker_initial_setup:\n\nInitial configuration\n---------------------\n\nWhen a Gel container starts on the specified data directory or remote\nPostgres cluster for the first time, initial instance setup is performed. This\nis called the *bootstrap phase*.\n\nThe following environment variables affect the bootstrap only and have no\neffect on subsequent container runs.\n\n.. note::\n\n For |EdgeDB| versions before 6.0 (Gel) the prefix for all environment\n variables is ``EDGEDB_`` instead of ``GEL_``.\n\n\nGEL_SERVER_BOOTSTRAP_COMMAND\n............................\n\nUseful to fine-tune initial user and branch creation, and other initial\nsetup. If neither the :gelenv:`SERVER_BOOTSTRAP_COMMAND` variable or the\n:gelenv:`SERVER_BOOTSTRAP_SCRIPT_FILE` are explicitly specified, the container\nwill look for the presence of ``/gel-bootstrap.edgeql`` in the container\n(which can be placed in a derived image).\n\nMaps directly to the |gel-server| flag ``--bootstrap-command``. The\n``*_FILE`` and ``*_ENV`` variants are also supported.\n\n\nGEL_SERVER_BOOTSTRAP_SCRIPT_FILE\n................................\nDeprecated in image version 2.8: use :gelenv:`SERVER_BOOTSTRAP_COMMAND_FILE`\ninstead.\n\nRun the script when initializing the database. The script is run by default\nuser within default branch.\n\n\nGEL_SERVER_PASSWORD\n...................\n\nThe password for the default superuser account will be set to this value. If\nno value is provided a password will not be set, unless set via\n:gelenv:`SERVER_BOOTSTRAP_COMMAND`. (If a value for\n:gelenv:`SERVER_BOOTSTRAP_COMMAND` is provided, this variable will be\nignored.)\n\nThe ``*_FILE`` and ``*_ENV`` variants are also supported.\n\n\nGEL_SERVER_PASSWORD_HASH\n........................\n\nA variant of :gelenv:`SERVER_PASSWORD`, where the specified value is a hashed\npassword verifier instead of plain text.\n\nIf :gelenv:`SERVER_BOOTSTRAP_COMMAND` is set, this variable will be ignored.\n\nThe ``*_FILE`` and ``*_ENV`` variants are also supported.\n\n\nGEL_SERVER_GENERATE_SELF_SIGNED_CERT\n....................................\n\n.. warning::\n\n Deprecated: use :gelenv:`SERVER_TLS_CERT_MODE=generate_self_signed`\n instead.\n\nSet this option to ``1`` to tell the server to automatically generate a\nself-signed certificate with key file in the :gelenv:`SERVER_DATADIR` (if\npresent, see below), and echo the certificate content in the logs. If the\ncertificate file exists, the server will use it instead of generating a new\none.\n\nSelf-signed certificates are usually used in development and testing, you\nshould likely provide your own certificate and key file with the variables\nbelow.\n\n\nGEL_SERVER_TLS_CERT/GEL_SERVER_TLS_KEY\n......................................\n\nThe TLS certificate and private key data, exclusive with\n:gelenv:`SERVER_TLS_CERT_MODE=generate_self_signed`.\n\nThe ``*_FILE`` and ``*_ENV`` variants are also supported.\n\n\nCustom scripts in \"/docker-entrypoint.d/\"\n.........................................\n\nTo perform additional initialization, a derived image may include one or more\nexecutable files in ``/docker-entrypoint.d/``, which will get executed by the\ncontainer entrypoint *before* any other processing takes place.\n\n\nRuntime configuration\n---------------------\n\nGEL_DOCKER_LOG_LEVEL\n....................\n\nDetermines the log verbosity level in the entrypoint script. Valid levels are\n``trace``, ``debug``, ``info``, ``warning``, and ``error``. The default is\n``info``.\n\n.. _ref_guide_deployment_docker_custom_bootstrap_scripts:\n\nCustom scripts in \"/gel-bootstrap.d/\" and \"/gel-bootstrap-late.d\"\n.................................................................\n\nTo perform additional initialization, a derived image may include one or more\n``*.edgeql`` or ``*.sh`` scripts, which are executed in addition to and\n*after* the initialization specified by the environment variables above or the\n``/gel-bootstrap.edgeql`` script. Parts in ``/gel-bootstrap.d`` are\nexecuted *before* any schema migrations are applied, and parts in\n``/gel-bootstrap-late.d`` are executed *after* the schema migration have\nbeen applied.\n\n.. note::\n\n Best practice for naming your script files when you will have multiple\n script files to run on bootstrap is to prepend the filenames with ``01-``,\n ``02-``, and so on to indicate your desired order of execution.\n\n\nConnecting your application\n===========================\n\nTo connect your application to the Gel instance, you'll need to provide\nconnection parameters. Gel client libraries can be configured using either\na DSN (connection string) or individual environment variables.\n\nObtaining connection parameters\n-------------------------------\n\nYour connection requires the following components:\n\n- **Host**: The container hostname or IP address. In Docker Compose, this is\n the service name (e.g., ``gel``). For standalone containers, use\n ``localhost`` if on the same host, or the container's IP/hostname.\n- **Port**: ``5656`` (the default Gel port, unless remapped with ``-p``)\n- **Username**: |admin| (the default superuser)\n- **Password**: The value of :gelenv:`SERVER_PASSWORD` you set when starting\n the container\n- **Branch**: |main| (the default branch)\n\nConstruct the DSN using these values:\n\n.. code-block:: bash\n\n $ GEL_DSN=\"gel://admin:@:5656\"\n\nFor a Docker Compose setup with the service named ``gel``:\n\n.. code-block:: bash\n\n $ GEL_DSN=\"gel://admin:secret@gel:5656\"\n\nObtaining the TLS certificate\n-----------------------------\n\nIf you configured Gel with ``GEL_SERVER_TLS_CERT_MODE=generate_self_signed``,\nyour application needs the certificate to connect securely.\n\nRetrieve the certificate from the running container:\n\n.. code-block:: bash\n\n $ docker exec cat /var/lib/gel/data/edbtlscert.pem\n\nOr using the Gel utility script:\n\n.. code-block:: bash\n\n $ docker exec \\\n gel-show-secrets.sh --format=raw GEL_SERVER_TLS_CERT\n\nAlternatively, retrieve it using the Gel CLI:\n\n.. code-block:: bash\n\n $ gel --dsn $GEL_DSN --tls-security insecure \\\n query \"SELECT sys::get_tls_certificate()\"\n\nIf you mounted a persistent volume at :gelenv:`SERVER_DATADIR`, the\ncertificate is also available at ``/edbtlscert.pem``.\n\nUsing in your application\n-------------------------\n\nSet these environment variables in your application container:\n\n.. code-block:: yaml\n\n # docker-compose.yaml example\n services:\n app:\n image: your-app\n environment:\n GEL_DSN: \"gel://admin:secret@gel:5656\"\n # For self-signed certificates:\n GEL_CLIENT_TLS_SECURITY: \"insecure\"\n # Or provide the CA certificate:\n # GEL_TLS_CA: \"\"\n\nFor production, we recommend providing the TLS certificate rather than\ndisabling TLS verification:\n\n.. code-block:: yaml\n\n services:\n app:\n image: your-app\n environment:\n GEL_DSN: \"gel://admin:${GEL_PASSWORD}@gel:5656\"\n GEL_TLS_CA_FILE: \"/certs/gel-ca.pem\"\n volumes:\n - ./certs:/certs:ro\n\nGel's client libraries will automatically read these environment variables.\n\nLocal development with the CLI\n------------------------------\n\nTo make your Gel container easier to work with during local development,\ncreate an alias using :gelcmd:`instance link`.\n\n.. note::\n\n The command groups :gelcmd:`instance` and :gelcmd:`project` are not\n intended to manage production instances.\n\nFrom your host machine, link to the container:\n\n.. code-block:: bash\n\n $ gel instance link \\\n --dsn gel://admin:secret@localhost:5656 \\\n --non-interactive \\\n --trust-tls-cert \\\n my_docker_instance\n\nYou can now refer to the instance using the alias ``my_docker_instance``.\nUse this alias wherever an instance name is expected:\n\n.. code-block:: bash\n\n $ gel -I my_docker_instance\n Gel x.x\n Type \\help for help, \\quit to quit.\n gel>\n\nOr apply migrations:\n\n.. code-block:: bash\n\n $ gel -I my_docker_instance migrate\n\n\nHealth Checks\n=============\n\nUsing an HTTP client, you can perform health checks to monitor the status of\nyour Gel instance. Learn how to use them with our :ref:`health checks guide\n`.\n" }, { "path": "docs/reference/running/deployment/fly_io.rst", "content": ".. _ref_guide_deployment_fly_io:\n\n======\nFly.io\n======\n\n:edb-alt-title: Deploying Gel to Fly.io\n\nIn this guide we show how to deploy Gel using a `Fly.io `_\nPostgreSQL cluster as the backend. The deployment consists of two apps: one\nrunning Postgres and the other running Gel.\n\n.. include:: ./note_cloud_reset_password.rst\n\nPrerequisites\n=============\n\n* Fly.io account\n* ``flyctl`` CLI (`install `_)\n\n.. _flyctl-install: https://fly.io/docs/getting-started/installing-flyctl/\n\n\nProvision a Fly.io app for Gel\n==============================\n\nEvery Fly.io app must have a globally unique name, including service VMs like\nPostgres and Gel. Pick a name and assign it to a local environment variable\ncalled ``EDB_APP``. In the command below, replace ``myorg-gel`` with a name\nof your choosing.\n\n.. code-block:: bash\n\n $ EDB_APP=myorg-gel\n $ flyctl apps create --name $EDB_APP\n New app created: myorg-gel\n\n\nNow let's use the ``read`` command to securely assign a value to the\n``PASSWORD`` environment variable.\n\n.. code-block:: bash\n\n $ echo -n \"> \" && read -s PASSWORD\n\n\nNow let's assign this password to a Fly `secret\n`_, plus a few other secrets that\nwe'll need. There are a couple more environment variables we need to set:\n\n.. code-block:: bash\n\n $ flyctl secrets set \\\n GEL_SERVER_PASSWORD=\"$PASSWORD\" \\\n GEL_SERVER_BACKEND_DSN_ENV=DATABASE_URL \\\n GEL_SERVER_TLS_CERT_MODE=generate_self_signed \\\n GEL_SERVER_PORT=8080 \\\n --app $EDB_APP\n Secrets are staged for the first deployment\n\nLet's discuss what's going on with all these secrets.\n\n- The :gelenv:`SERVER_BACKEND_DSN_ENV` tells the Gel container where to\n look for the PostgreSQL connection string (more on that below)\n- The :gelenv:`SERVER_TLS_CERT_MODE` tells Gel to auto-generate a\n self-signed TLS certificate.\n\n You may instead choose to provision a custom TLS certificate. In this\n case, you should instead create two other secrets: assign your certificate\n to :gelenv:`SERVER_TLS_CERT` and your private key to\n :gelenv:`SERVER_TLS_KEY`.\n- Lastly, :gelenv:`SERVER_PORT` tells Gel to listen on port 8080 instead\n of the default 5656, because Fly.io prefers ``8080`` for its default health\n checks.\n\nFinally, let's configure the VM size as Gel requires a little bit more than\nthe default Fly.io VM side provides. Put this in a file called ``fly.toml`` in\nyour current directory.:\n\n.. code-block:: yaml\n\n [build]\n image = \"geldata/gel\"\n\n [[vm]]\n memory = \"512mb\"\n cpus = 1\n cpu-kind = \"shared\"\n\n\nCreate a PostgreSQL cluster\n===========================\n\nNow we need to provision a PostgreSQL cluster and attach it to the Gel app.\n\n.. note::\n\n If you have an existing PostgreSQL cluster in your Fly.io organization,\n you can skip to the attachment step.\n\nThen create a new PostgreSQL cluster. This may take a few minutes to complete.\n\n.. code-block:: bash\n\n $ PG_APP=myorg-postgres\n $ flyctl pg create --name $PG_APP --vm-size shared-cpu-1x\n ? Select region: sea (Seattle, Washington (US))\n ? Specify the initial cluster size: 1\n ? Volume size (GB): 10\n Creating postgres cluster myorg-postgres in organization personal\n Postgres cluster myorg-postgres created\n Username: postgres\n Password: \n Hostname: myorg-postgres.internal\n Proxy Port: 5432\n PG Port: 5433\n Save your credentials in a secure place, you won't be able to see them\n again!\n Monitoring Deployment\n ...\n --> v0 deployed successfully\n\nIn the output, you'll notice a line that says ``Machine is\ncreated``. The ID in that line is the ID of the virtual machine created for\nyour Postgres cluster. We now need to use that ID to scale the cluster since\nthe ``shared-cpu-1x`` VM doesn't have enough memory by default. Scale it with\nthis command:\n\n.. code-block:: bash\n\n $ flyctl machine update --memory 1024 --app $PG_APP -y\n Searching for image 'flyio/postgres:14.6' remotely...\n image found: img_0lq747j0ym646x35\n Image: registry-1.docker.io/flyio/postgres:14.6\n Image size: 361 MB\n\n Updating machine \n Waiting for to become healthy (started, 3/3)\n Machine updated successfully!\n ==> Monitoring health checks\n Waiting for to become healthy (started, 3/3)\n ...\n\nWith the VM scaled sufficiently, we can now attach the PostgreSQL cluster to\nthe Gel app:\n\n.. code-block:: bash\n\n $ PG_ROLE=myorg_gel\n $ flyctl pg attach \"$PG_APP\" \\\n --database-user \"$PG_ROLE\" \\\n --app $EDB_APP\n Postgres cluster myorg-postgres is now attached to myorg-gel\n The following secret was added to myorg-gel:\n DATABASE_URL=postgres://...\n\nLastly, Gel needs the ability to create Postgres databases and roles,\nso let's adjust the permissions on the role that Gel will use to connect\nto Postgres:\n\n.. code-block:: bash\n\n $ echo \"alter role \\\"$PG_ROLE\\\" createrole createdb; \\quit\" \\\n | flyctl pg connect --app $PG_APP\n ...\n ALTER ROLE\n\n.. _ref_guide_deployment_fly_io_start_gel:\n\nStart Gel\n=========\n\nEverything is set! Time to start Gel.\n\n.. code-block:: bash\n\n $ flyctl deploy --remote-only --app $EDB_APP\n ...\n Finished launching new machines\n -------\n āœ” Machine e286630dce9638 [app] was created\n -------\n\nThat's it! You can now start using the Gel instance located at\n:geluri:`myorg-gel.internal` in your Fly.io apps.\n\n\nIf deploy did not succeed:\n\n1. make sure you've created the ``fly.toml`` file.\n2. re-run the ``deploy`` command\n3. check the logs for more information: ``flyctl logs --app $EDB_APP``\n\nPersist the generated TLS certificate\n=====================================\n\nNow we need to persist the auto-generated TLS certificate to make sure it\nsurvives Gel app restarts. (If you've provided your own certificate,\nskip this step).\n\n.. code-block:: bash\n\n $ EDB_SECRETS=\"GEL_SERVER_TLS_KEY GEL_SERVER_TLS_CERT\"\n $ flyctl ssh console --app $EDB_APP -C \\\n \"gel-show-secrets.sh --format=toml $EDB_SECRETS\" \\\n | tr -d '\\r' | flyctl secrets import --app $EDB_APP\n\n\nConnecting your application\n===========================\n\nTo connect your application to the Gel instance, you'll need to provide\nconnection parameters. Gel client libraries can be configured using either\na DSN (connection string) or individual environment variables.\n\nObtaining connection parameters\n-------------------------------\n\nYour connection requires the following components:\n\n- **Host (internal)**: ``$EDB_APP.internal`` ā€” Fly uses this synthetic TLD\n for inter-app communication (e.g., ``myorg-gel.internal``)\n- **Host (external)**: ``$EDB_APP.fly.dev`` ā€” for connections from outside\n Fly.io (requires exposing the port, see below)\n- **Port**: ``8080``, which we configured earlier with :gelenv:`SERVER_PORT`\n- **Username**: |admin| (the default superuser)\n- **Password**: The value you assigned to ``$PASSWORD``\n- **Branch**: |main| (the default branch)\n\nConstruct the DSN for internal Fly.io connections:\n\n.. code-block:: bash\n\n $ GEL_DSN=gel://admin:$PASSWORD@$EDB_APP.internal:8080\n\nConsider writing it to a file to ensure the DSN looks correct. Remember to\ndelete the file after you're done. (Printing this value to the terminal with\n``echo`` is insecure and can leak your password into shell logs.)\n\n.. code-block:: bash\n\n $ echo $GEL_DSN > dsn.txt\n $ open dsn.txt\n $ rm dsn.txt\n\nObtaining the TLS certificate\n-----------------------------\n\nIf you need secure TLS connections (required for external access), retrieve\nthe server's TLS certificate:\n\n.. code-block:: bash\n\n $ flyctl ssh console -a $EDB_APP \\\n -C \"gel-show-secrets.sh --format=raw GEL_SERVER_TLS_CERT\"\n\nSave this to a file or set it as a secret in your application.\n\nFrom a Fly.io app\n-----------------\n\nTo connect to this instance from another Fly app (say, an app that runs your\nAPI server) set the value of the :gelenv:`DSN` secret inside that app.\n\n.. code-block:: bash\n\n $ flyctl secrets set \\\n GEL_DSN=$DSN \\\n --app my-other-fly-app\n\nWe'll also set another variable that will disable Gel's TLS checks.\nInter-application communication is secured by Fly so TLS isn't vital in\nthis case; configuring TLS certificates is also beyond the scope of this guide.\n\n.. code-block:: bash\n\n $ flyctl secrets set GEL_CLIENT_TLS_SECURITY=insecure \\\n --app my-other-fly-app\n\n\nYou can also set these values as environment variables inside your\n``fly.toml`` file, but using Fly's built-in `secrets\n`_ functionality is recommended.\n\nFrom external application\n-------------------------\n\nIf you need to access Gel from outside the Fly.io network, you'll need to\nconfigure the Fly.io proxy to let external connections in.\n\nLet's make sure the ``[[services]]`` section in our ``fly.toml`` looks\nsomething like this:\n\n.. code-block:: toml\n\n [[services]]\n http_checks = []\n internal_port = 8080\n processes = [\"app\"]\n protocol = \"tcp\"\n script_checks = []\n [services.concurrency]\n hard_limit = 25\n soft_limit = 20\n type = \"connections\"\n\n [[services.ports]]\n port = 5656\n\n [[services.tcp_checks]]\n grace_period = \"1s\"\n interval = \"15s\"\n restart_limit = 0\n timeout = \"2s\"\n\nIn the same directory, :ref:`redeploy the Gel app\n`. This makes the Gel port\navailable to the outside world. You can now access the instance from any host\nvia the following public DSN: :geluri:`admin:$PASSWORD@$EDB_APP.fly.dev`.\n\nTo secure communication between the server and the client, you will also\nneed to set the :gelenv:`TLS_CA` environment secret in your application.\nYou can securely obtain the certificate content by running:\n\n.. code-block:: bash\n\n $ flyctl ssh console -a $EDB_APP \\\n -C \"gel-show-secrets.sh --format=raw GEL_SERVER_TLS_CERT\"\n\nLocal development with the CLI\n------------------------------\n\nTo access the Gel instance from your local development machine, install\nthe Wireguard `VPN `_ and create a tunnel, as described on Fly's\n`Private Networking\n`_\ndocs.\n\nOnce it's up and running, use :gelcmd:`instance link` to create a local\nalias to the remote instance.\n\n.. note::\n\n The command groups :gelcmd:`instance` and :gelcmd:`project` are not\n intended to manage production instances.\n\n.. code-block:: bash\n\n $ gel instance link \\\n --dsn $GEL_DSN \\\n --non-interactive \\\n --trust-tls-cert \\\n my_fly_instance\n Authenticating to gel://admin@myorg-gel.internal:8080/main\n Successfully linked to remote instance. To connect run:\n gel -I my_fly_instance\n\nYou can now refer to the remote instance using the alias ``my_fly_instance``.\nUse this alias wherever an instance name is expected:\n\n.. code-block:: bash\n\n $ gel -I my_fly_instance\n Gel x.x\n Type \\help for help, \\quit to quit.\n gel>\n\nOr apply migrations:\n\n.. code-block:: bash\n\n $ gel -I my_fly_instance migrate\n\n.. _vpn: https://fly.io/docs/reference/private-networking/#private-network-vpn\n\n\nHealth Checks\n=============\n\nUsing an HTTP client, you can perform health checks to monitor the status of\nyour Gel instance. Learn how to use them with our :ref:`health checks guide\n`.\n" }, { "path": "docs/reference/running/deployment/gcp.rst", "content": ".. _ref_guide_deployment_gcp:\n\n============\nGoogle Cloud\n============\n\n:edb-alt-title: Deploying Gel to Google Cloud\n\nIn this guide we show how to deploy Gel on GCP using Cloud SQL and\nKubernetes.\n\n.. include:: ./note_cloud_reset_password.rst\n\nPrerequisites\n=============\n\n* Google Cloud account with billing enabled (or a `free trial `_)\n* ``gcloud`` CLI (`install `_)\n* ``kubectl`` CLI (`install `_)\n\n.. _gcp-trial: https://cloud.google.com/free/\n.. _gcloud-intsll: https://cloud.google.com/sdk/\n.. _kubectl-install: https://kubernetes.io/docs/tasks/tools/install-kubectl/\n\nMake sure you are logged into Google Cloud.\n\n.. code-block:: bash\n\n $ gcloud init\n\nCreate a project\n================\n\nSet the ``PROJECT`` environment variable to the project name you'd like to\nuse. Google Cloud only allow letters, numbers, and hyphens.\n\n.. code-block:: bash\n\n $ PROJECT=gel\n\nThen create a project with this name. Skip this step if your project already\nexists.\n\n.. code-block:: bash\n\n $ gcloud projects create $PROJECT\n\nThen enable the requisite APIs.\n\n.. code-block:: bash\n\n $ gcloud services enable \\\n container.googleapis.com \\\n sqladmin.googleapis.com \\\n iam.googleapis.com \\\n --project=$PROJECT\n\nProvision a Postgres instance\n=============================\n\nUse the ``read`` command to securely assign a value to the ``PASSWORD``\nenvironment variable.\n\n.. code-block:: bash\n\n $ echo -n \"> \" && read -s PASSWORD\n\nThen create a Cloud SQL instance and set the password.\n\n.. code-block:: bash\n\n $ gcloud sql instances create ${PROJECT}-postgres \\\n --database-version=POSTGRES_17 \\\n --edition=enterprise \\\n --cpu=1 \\\n --memory=3840MiB \\\n --region=us-west2 \\\n --project=$PROJECT\n $ gcloud sql users set-password postgres \\\n --instance=${PROJECT}-postgres \\\n --password=$PASSWORD \\\n --project=$PROJECT\n\nCreate a Kubernetes cluster\n===========================\n\nCreate an empty Kubernetes cluster inside your project.\n\n.. code-block:: bash\n\n $ gcloud container clusters create ${PROJECT}-k8s \\\n --zone=us-west2-a \\\n --num-nodes=1 \\\n --project=$PROJECT\n\nConfigure service account\n=========================\n\nCreate a new service account, configure its permissions, and generate a\n``credentials.json`` file.\n\n.. code-block:: bash\n\n $ gcloud iam service-accounts create ${PROJECT}-account \\\n --project=$PROJECT\n\n $ MEMBER=\"${PROJECT}-account@${PROJECT}.iam.gserviceaccount.com\"\n $ gcloud projects add-iam-policy-binding $PROJECT \\\n --member=serviceAccount:${MEMBER} \\\n --role=roles/cloudsql.admin \\\n --project=$PROJECT\n\n $ gcloud iam service-accounts keys create credentials.json \\\n --iam-account=${MEMBER}\n\nThen use this ``credentials.json`` to authenticate the Kubernetes CLI tool\n``kubectl``.\n\n.. code-block:: bash\n\n $ gcloud components install gke-gcloud-auth-plugin\n $ kubectl create secret generic cloudsql-instance-credentials \\\n --from-file=credentials.json=credentials.json\n\n $ INSTANCE_CONNECTION_NAME=$(\n gcloud sql instances describe ${PROJECT}-postgres \\\n --format=\"value(connectionName)\" \\\n --project=$PROJECT\n )\n\n $ DSN=\"postgresql://postgres:${PASSWORD}@127.0.0.1:5432\"\n $ kubectl create secret generic cloudsql-db-credentials \\\n --from-literal=dsn=$DSN \\\n --from-literal=password=$PASSWORD \\\n --from-literal=instance=${INSTANCE_CONNECTION_NAME}=tcp:5432\n\nDeploy Gel\n==========\n\nDownload the starter Gel Kubernetes configuration file. This file specifies\na persistent volume, a container running a `Cloud SQL authorization proxy\n`_, and a container to\nrun `Gel itself `_. It relies on\nthe secrets we declared in the previous step.\n\n.. code-block:: bash\n\n $ wget \"https://raw.githubusercontent.com\\\n /geldata/gel-deploy/dev/gcp/deployment.yaml\"\n\n $ kubectl apply -f deployment.yaml\n\nEnsure the pods are running.\n\n.. code-block:: bash\n\n $ kubectl get pods\n NAME READY STATUS RESTARTS AGE\n gel-977b8fdf6-jswlw 0/2 ContainerCreating 0 16s\n\nThe ``READY 0/2`` tells us neither of the two pods have finished booting.\nRe-run the command until ``2/2`` pods are ``READY``.\n\nIf there were errors you can check Gel's logs with:\n\n.. code-block:: bash\n\n $ kubectl logs deployment/gel --container gel\n\nPersist TLS Certificate\n=======================\n\nNow that our Gel instance is up and running, we need to download a local\ncopy of its self-signed TLS certificate (which it generated on startup) and\npass it as a secret into Kubernetes. Then we'll redeploy the pods.\n\n.. code-block:: bash\n\n $ kubectl create secret generic cloudsql-tls-credentials \\\n --from-literal=tlskey=\"$(\n kubectl exec deploy/gel -c=gel -- \\\n gel-show-secrets.sh --format=raw GEL_SERVER_TLS_KEY\n )\" \\\n --from-literal=tlscert=\"$(\n kubectl exec deploy/gel -c=gel -- \\\n gel-show-secrets.sh --format=raw GEL_SERVER_TLS_CERT\n )\"\n\n $ kubectl delete -f deployment.yaml\n\n $ kubectl apply -f deployment.yaml\n\nExpose Gel\n==========\n\n.. code-block:: bash\n\n $ kubectl expose deploy/gel --type LoadBalancer\n\n\nConnecting your application\n===========================\n\nTo connect your application to the Gel instance, you'll need to provide\nconnection parameters. Gel client libraries can be configured using either\na DSN (connection string) or individual environment variables.\n\nObtaining connection parameters\n-------------------------------\n\nYour connection requires the following components:\n\n- **Host**: The ``EXTERNAL-IP`` of the LoadBalancer service\n- **Port**: ``5656`` (the default Gel port)\n- **Username**: |admin| (the default superuser)\n- **Password**: The value you assigned to ``$PASSWORD``\n- **Branch**: |main| (the default branch)\n\nGet the public-facing IP address of your database:\n\n.. code-block:: bash\n\n $ kubectl get service\n NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S)\n gel LoadBalancer 5656:30841/TCP\n\nCopy the ``EXTERNAL-IP`` associated with the ``gel`` service and construct\nyour instance's :ref:`DSN `:\n\n.. code-block:: bash\n\n $ GEL_IP=\n $ GEL_DSN=\"gel://admin:${PASSWORD}@${GEL_IP}\"\n\nTo print the final DSN, you can ``echo`` it. Note that you should only run\nthis command on a computer you trust, like a personal laptop or sandboxed\nenvironment.\n\n.. code-block:: bash\n\n $ echo $GEL_DSN\n\nObtaining the TLS certificate\n-----------------------------\n\nSince we configured Gel with a self-signed TLS certificate, your application\nneeds the certificate to connect securely. Retrieve it from the running pod:\n\n.. code-block:: bash\n\n $ kubectl exec deploy/gel -c=gel -- \\\n gel-show-secrets.sh --format=raw GEL_SERVER_TLS_CERT \\\n > gel-tls-cert.pem\n\nAlternatively, retrieve it using the Gel CLI:\n\n.. code-block:: bash\n\n $ gel --dsn $GEL_DSN --tls-security insecure \\\n query \"SELECT sys::get_tls_certificate()\" > gel-tls-cert.pem\n\nTest your connection by opening a REPL:\n\n.. code-block:: bash\n\n $ gel --dsn $GEL_DSN --tls-security insecure\n Gel x.x (repl x.x)\n Type \\help for help, \\quit to quit.\n gel> select \"hello world!\";\n\nLocal development with the CLI\n------------------------------\n\nTo make your remote instance easier to work with during local development,\ncreate an alias using :gelcmd:`instance link`.\n\n.. note::\n\n The command groups :gelcmd:`instance` and :gelcmd:`project` are not\n intended to manage production instances.\n\n.. code-block:: bash\n\n $ echo $PASSWORD | gel instance link \\\n --dsn $GEL_DSN \\\n --password-from-stdin \\\n --non-interactive \\\n --trust-tls-cert \\\n my_gcp_instance\n\nYou can now refer to the remote instance using the alias ``my_gcp_instance``.\nUse this alias wherever an instance name is expected:\n\n.. code-block:: bash\n\n $ gel -I my_gcp_instance\n Gel x.x\n Type \\help for help, \\quit to quit.\n gel>\n\nOr apply migrations:\n\n.. code-block:: bash\n\n $ gel -I my_gcp_instance migrate\n\nUsing in your application\n-------------------------\n\nSet these environment variables where you deploy your application:\n\n.. code-block:: bash\n\n GEL_DSN=\"gel://admin:@:5656\"\n # For self-signed certificates, provide the CA cert:\n GEL_TLS_CA_FILE=\"/path/to/gel-tls-cert.pem\"\n # Or embed the certificate content directly:\n GEL_TLS_CA=\"\"\n # Or (for development only) disable TLS verification:\n # GEL_CLIENT_TLS_SECURITY=insecure\n\nGel's client libraries will automatically read these environment variables.\n\n\nHealth Checks\n=============\n\nUsing an HTTP client, you can perform health checks to monitor the status of\nyour Gel instance. Learn how to use them with our :ref:`health checks guide\n`.\n" }, { "path": "docs/reference/running/deployment/index.rst", "content": ".. _ref_guide_deployment:\n\n\n==========\nDeployment\n==========\n\n|Gel| can be hosted on all major cloud hosting platforms. The guides below\ndemonstrate how to spin up both a managed PostgreSQL instance and a container\nrunning Gel `in Docker `_.\n\n.. note:: Minimum requirements\n\n As a rule of thumb, the Gel Docker container requires 1GB RAM! Images\n with insufficient RAM may experience unexpected issues during startup.\n\n When using an external PostgreSQL instance Gel must connect with the\n PostgreSQL superuser.\n\n.. toctree::\n :maxdepth: 1\n\n aws_aurora_ecs\n azure_flexibleserver\n digitalocean\n fly_io\n gcp\n docker\n bare_metal\n" }, { "path": "docs/reference/running/deployment/note_cloud_reset_password.rst", "content": ".. note:: Gel Cloud: Reset the default password for the admin role\n\n If you want to dump an existing Gel Cloud instance and restore it to a new self-managed instance, you need to change the automatically generated password for the default admin role - ``edgedb`` or ``admin``.\n The administrator role name and its password used in the dump/restore process must be the same in both the instance dumped from and the instance restored to for the Gel tooling to continue functioning properly.\n To change the default password in the Cloud instance, execute the following query in the instance:\n\n .. code-block:: edgeql\n\n ALTER ROLE admin { set password := 'new_password' };\n" }, { "path": "docs/reference/running/http.rst", "content": ".. _ref_reference_http_api:\n\n========\nHTTP API\n========\n\nGel provides HTTP endpoints that allow you to monitor the health and performance of your instance. You can use these endpoints to check if your instance is alive and ready to receive queries, as well as to collect metrics about its operation.\n\nYour branch's URL takes the form of ``http://:``.\n\nHere's how to determine your local Gel instance's HTTP server URL:\n\n- The ``hostname`` will be ``localhost``\n\n- Find the ``port`` by running :gelcmd:`instance list`. This will print a table of all Gel instances on your machine, including their associated port number.\n\nTo determine the URL of a remote instance you have linked with the CLI, you can get both the hostname and port of the instance from the \"Port\" column of the :gelcmd:`instance list` table (formatted as ``:``).\n\n.. _ref_edgeql_http_health_checks:\n.. _ref_reference_health_checks:\n.. _ref_guide_deployment_health_checks:\n\nHealth Checks\n=============\n\n|Gel| exposes endpoints to check for aliveness and readiness of your database\ninstance.\n\nAliveness\n---------\n\nCheck if your instance is alive.\n\n.. code-block::\n\n http://:/server/status/alive\n\nIf your instance is alive, it will respond with a ``200`` status code and ``\"OK\"`` as the payload. Otherwise, it will respond with a ``50x`` or a network error.\n\nReadiness\n---------\n\nCheck if your instance is ready to receive queries.\n\n.. code-block::\n\n http://:/server/status/ready\n\nIf your instance is ready, it will respond with a ``200`` status code and ``\"OK\"`` as the payload. Otherwise, it will respond with a ``50x`` or a network error.\n\n\n.. _ref_observability:\n\nObservability\n=============\n\nRetrieve instance metrics.\n\n.. code-block::\n\n http://:/metrics\n\nAll Gel instances expose a Prometheus-compatible endpoint available via GET request. The following metrics are made available.\n\nSystem\n------\n\n``compiler_process_spawns_total``\n **Counter.** Total number of compiler processes spawned.\n\n``compiler_processes_current``\n **Gauge.** Current number of active compiler processes.\n\n``branches_current``\n **Gauge.** Current number of branches.\n\nBackend connections and performance\n-----------------------------------\n\n``backend_connections_total``\n **Counter.** Total number of backend connections established.\n\n``backend_connections_current``\n **Gauge.** Current number of active backend connections.\n\n``backend_connection_establishment_errors_total``\n **Counter.** Number of times the server could not establish a backend connection.\n\n``backend_connection_establishment_latency``\n **Histogram.** Time it takes to establish a backend connection, in seconds.\n\n``backend_query_duration``\n **Histogram.** Time it takes to run a query on a backend connection, in seconds.\n\nClient connections\n------------------\n\n``client_connections_total``\n **Counter.** Total number of clients.\n\n``client_connections_current``\n **Gauge.** Current number of active clients.\n\n``client_connections_idle_total``\n **Counter.** Total number of forcefully closed idle client connections.\n\n``client_connection_duration``\n **Histogram.** Time a client connection is open.\n\nQueries and compilation\n-----------------------\n\n``edgeql_query_compilations_total``\n **Counter.** Number of compiled/cached queries or scripts since instance startup. A query is compiled and then cached on first use, increasing the ``path=\"compiler\"`` parameter. Subsequent uses of the same query only use the cache, thus only increasing the ``path=\"cache\"`` parameter.\n\n``edgeql_query_compilation_duration``\n Deprecated in favor of ``query_compilation_duration[interface=\"edgeql\"]``.\n\n **Histogram.** Time it takes to compile an EdgeQL query or script, in seconds.\n\n``graphql_query_compilations_total``\n **Counter.** Number of compiled/cached GraphQL queries since instance startup. A query is compiled and then cached on first use, increasing the ``path=\"compiler\"`` parameter. Subsequent uses of the same query only use the cache, thus only increasing the ``path=\"cache\"`` parameter.\n\n``sql_queries_total``\n **Counter.** Number of SQL queries since instance startup.\n\n``sql_compilations_total``\n **Counter.** Number of SQL compilations since instance startup.\n\n``query_compilation_duration``\n **Histogram.** Time it takes to compile a query or script, in seconds.\n\n``queries_per_connection``\n **Histogram.** Number of queries per connection.\n\n``query_size``\n **Histogram.** Number of bytes in a query, where the label ``interface=edgeql`` means the size of an EdgeQL query, ``=graphql`` for a GraphQL query, ``=sql`` for a readonly SQL query from the user, and ``=compiled`` for a backend SQL query compiled and issued by the server.\n\nAuth Extension\n--------------\n\n``auth_api_calls_total``\n **Counter.** Number of API calls to the Auth extension.\n\n``auth_ui_renders_total``\n **Counter.** Number of UI pages rendered by the Auth extension.\n\n``auth_providers``\n **Histogram.** Number of Auth providers configured.\n\n``auth_successful_logins_total``\n **Counter.** Number of successful logins in the Auth extension.\n\nErrors\n------\n\n``background_errors_total``\n **Counter.** Number of unhandled errors in background server routines.\n\n``transaction_serialization_errors_total``\n **Counter.** Number of transaction serialization errors.\n\n``connection_errors_total``\n **Counter.** Number of network connection errors." }, { "path": "docs/reference/running/index.rst", "content": ".. _ref_running_index:\n\n===========\nRunning Gel\n===========\n\n.. toctree::\n :maxdepth: 2\n :hidden:\n\n local\n deployment/index\n configuration\n http\n backend_ha\n admin/index\n\nThis section provides comprehensive guidance for deploying and managing Gel database instances.\n\nGet your instance running\n=========================\n\nWhile running local project instances with the CLI is low maintenance and easy to get started, Gel also makes it easy to configure your production deployment through:\n\n* Environment variables\n* Configuration files\n* Server CLI arguments\n\nThese configuration mechanisms allow you to tailor your Gel deployment to your specific needs. Operations teams can control everything from memory usage, connection limits, to TLS and network settings, ensuring your Gel deployment aligns with organizational policies and infrastructure constraints while maintaining optimal performance.\n\nAnd keep it humming\n===================\n\nRunning Gel in production requires effective management and monitoring capabilities. Gel provides a comprehensive set of tools for these purposes:\n\n- **HTTP endpoints** for health checks and metrics collection that integrate with your monitoring infrastructure\n- **Administrative commands** for:\n - Role management (creating, altering, and dropping roles)\n - Branch operations (creating, dropping, and managing branches)\n - Database maintenance operations like vacuum for reclaiming space\n - Updating internal statistics used by the query planner\n\nThese tools enable you to maintain optimal performance, manage access control, and troubleshoot issues in your Gel deployments. The reference documentation for these features is organized in the sections that follow.\n" }, { "path": "docs/reference/running/local.rst", "content": ".. _ref_running_local:\n\n=================\nLocal development\n=================\n\nSee the :ref:`ref_guide_using_projects` page for information on how to use projects to manage local development." }, { "path": "docs/reference/stdlib/abstract.rst", "content": ".. _ref_std_abstract_types:\n\n==============\nAbstract Types\n==============\n\nAbstract types are used to describe polymorphic functions, otherwise known as\n\"generic functions,\" which can be called on a broad range of types.\n\n\n----------\n\n\n.. eql:type:: anytype\n\n :index: any anytype\n\n A generic type.\n\n It is a placeholder used in cases where no specific type\n requirements are needed, such as defining polymorphic parameters\n in functions and operators.\n\n\n----------\n\n\n.. eql:type:: std::anyobject\n\n :index: any anytype object\n\n A generic object.\n\n Similarly to :eql:type:`anytype`, this type is used to denote a generic\n object. This is useful when defining polymorphic parameters in functions\n and operators as it conforms to whatever type is actually passed. This is\n different friom :eql:type:`BaseObject` which although is the parent type\n of any object also only has an ``id`` property, making access to other\n properties and links harder.\n\n\n----------\n\n\n.. eql:type:: std::anyscalar\n\n :index: any anytype scalar\n\n An abstract base scalar type.\n\n All scalar types are derived from this type.\n\n\n----------\n\n\n.. eql:type:: std::anyenum\n\n :index: any anytype enum\n\n An abstract base enumerated type.\n\n All :eql:type:`enum` types are derived from this type.\n\n\n----------\n\n\n.. eql:type:: anytuple\n\n :index: any anytype anytuple\n\n A generic tuple.\n\n Similarly to :eql:type:`anytype`, this type is used to denote a generic\n tuple without detailing its component types. This is useful when defining\n polymorphic parameters in functions and operators.\n\n\nAbstract Numeric Types\n======================\n\nThese abstract numeric types extend :eql:type:`anyscalar`.\n\n.. eql:type:: std::anyint\n\n :index: any anytype int\n\n An abstract base scalar type for\n :eql:type:`int16`, :eql:type:`int32`, and :eql:type:`int64`.\n\n\n----------\n\n\n.. eql:type:: std::anyfloat\n\n :index: any anytype float\n\n An abstract base scalar type for\n :eql:type:`float32` and :eql:type:`float64`.\n\n\n----------\n\n\n.. eql:type:: std::anyreal\n\n :index: any anytype\n\n An abstract base scalar type for\n :eql:type:`anyint`, :eql:type:`anyfloat`, and :eql:type:`decimal`.\n\n\nAbstract Range Types\n====================\n\nThere are some types that can be used to construct :ref:`ranges\n`. These scalar types are distinguished by the following\nabstract types:\n\n.. eql:type:: std::anypoint\n\n :index: any anypoint anyrange\n\n Abstract base type for all valid ranges.\n\n Abstract base scalar type for :eql:type:`int32`, :eql:type:`int64`,\n :eql:type:`float32`, :eql:type:`float64`, :eql:type:`decimal`,\n :eql:type:`datetime`, :eql:type:`cal::local_datetime`, and\n :eql:type:`cal::local_date`.\n\n\n----------\n\n\n.. eql:type:: std::anydiscrete\n\n :index: any anydiscrete anyrange discrete\n\n An abstract base type for all valid *discrete* ranges.\n\n This is an abstract base scalar type for :eql:type:`int32`,\n :eql:type:`int64`, and :eql:type:`cal::local_date`.\n\n\n----------\n\n\n.. eql:type:: std::anycontiguous\n\n :index: any anycontiguous anyrange\n\n An abstract base type for all valid *contiguous* ranges.\n\n This is an abstract base scalar type for :eql:type:`float32`,\n :eql:type:`float64`, :eql:type:`decimal`, :eql:type:`datetime`, and\n :eql:type:`cal::local_datetime`.\n" }, { "path": "docs/reference/stdlib/array.rst", "content": ".. _ref_std_array:\n\n======\nArrays\n======\n\n:edb-alt-title: Array Functions and Operators\n\n.. list-table::\n :class: funcoptable\n\n * - :eql:op:`array[i] `\n - :eql:op-desc:`arrayidx`\n\n * - :eql:op:`array[from:to] `\n - :eql:op-desc:`arrayslice`\n\n * - :eql:op:`array ++ array `\n - :eql:op-desc:`arrayplus`\n\n * - :eql:op:`= ` :eql:op:`\\!= ` :eql:op:`?= `\n :eql:op:`?!= ` :eql:op:`\\< ` :eql:op:`\\> `\n :eql:op:`\\<= ` :eql:op:`\\>= `\n - Comparison operators\n\n * - :eql:func:`len`\n - Returns the number of elements in the array.\n\n * - :eql:func:`contains`\n - Checks if an element is in the array.\n\n * - :eql:func:`find`\n - Finds the index of an element in the array.\n\n * - :eql:func:`array_join`\n - Renders an array to a string or byte-string.\n\n * - :eql:func:`array_fill`\n - :eql:func-desc:`array_fill`\n\n * - :eql:func:`array_replace`\n - :eql:func-desc:`array_replace`\n\n * - :eql:func:`array_set`\n - :eql:func-desc:`array_set`\n\n * - :eql:func:`array_insert`\n - :eql:func-desc:`array_insert`\n\n * - :eql:func:`array_agg`\n - :eql:func-desc:`array_agg`\n\n * - :eql:func:`array_get`\n - :eql:func-desc:`array_get`\n\n * - :eql:func:`array_unpack`\n - :eql:func-desc:`array_unpack`\n\nArrays store expressions of the *same type* in an ordered list.\n\n.. _ref_std_array_constructor:\n\nConstructing arrays\n^^^^^^^^^^^^^^^^^^^\n\nAn array constructor is an expression that consists of a sequence of\ncomma-separated expressions *of the same type* enclosed in square brackets.\nIt produces an array value:\n\n.. eql:synopsis::\n\n \"[\" [, ...] \"]\"\n\nFor example:\n\n.. code-block:: edgeql-repl\n\n db> select [1, 2, 3];\n {[1, 2, 3]}\n db> select [('a', 1), ('b', 2), ('c', 3)];\n {[('a', 1), ('b', 2), ('c', 3)]}\n\nEmpty arrays\n^^^^^^^^^^^^\n\nYou can also create an empty array, but it must be done by providing the type\ninformation using type casting. Gel cannot infer the type of an empty array\ncreated otherwise. For example:\n\n.. code-block:: edgeql-repl\n\n db> select [];\n QueryError: expression returns value of indeterminate type\n Hint: Consider using an explicit type cast.\n ### select [];\n ### ^\n\n db> select >[];\n {[]}\n\n\n\nReference\n^^^^^^^^^\n\n.. eql:type:: std::array\n\n An ordered list of values of the same type.\n\n Array indexing starts at zero.\n\n An array can contain any type except another array. In Gel, arrays are\n always one-dimensional.\n\n An array type is created implicitly when an :ref:`array\n constructor ` is used:\n\n .. code-block:: edgeql-repl\n\n db> select [1, 2];\n {[1, 2]}\n\n The array types themselves are denoted by ``array`` followed by their\n sub-type in angle brackets. These may appear in cast operations:\n\n .. code-block:: edgeql-repl\n\n db> select >[1, 4, 7];\n {['1', '4', '7']}\n db> select >[1, 4, 7];\n {[1n, 4n, 7n]}\n\n Array types may also appear in schema declarations:\n\n .. code-block:: sdl\n\n type Person {\n str_array: array;\n json_array: array;\n }\n\n See also the list of standard :ref:`array functions `, as\n well as :ref:`generic functions ` such as\n :eql:func:`len`.\n\n\n----------\n\n\n.. eql:operator:: arrayidx: array [ int64 ] -> anytype\n\n .. api-index:: Ā§arrayĀ§[Ā§intĀ§]\n\n Accesses the array element at a given index.\n\n Example:\n\n .. code-block:: edgeql-repl\n\n db> select [1, 2, 3][0];\n {1}\n db> select [(x := 1, y := 1), (x := 2, y := 3.3)][1];\n {(x := 2, y := 3.3)}\n\n This operator also allows accessing elements from the end of the array\n using a negative index:\n\n .. code-block:: edgeql-repl\n\n db> select [1, 2, 3][-1];\n {3}\n\n Referencing a non-existent array element will result in an error:\n\n .. code-block:: edgeql-repl\n\n db> select [1, 2, 3][4];\n InvalidValueError: array index 4 is out of bounds\n\n\n----------\n\n\n.. eql:operator:: arrayslice: array [ int64 : int64 ] -> anytype\n\n .. api-index:: Ā§arrayĀ§[Ā§intĀ§:Ā§intĀ§]\n\n Produces a sub-array from an existing array.\n\n Omitting the lower bound of an array slice will default to a lower bound\n of zero.\n\n Omitting the upper bound will default the upper bound to the length of the\n array.\n\n The lower bound of an array slice is inclusive while the upper bound is\n not.\n\n Examples:\n\n .. code-block:: edgeql-repl\n\n db> select [1, 2, 3][0:2];\n {[1, 2]}\n db> select [1, 2, 3][2:];\n {[3]}\n db> select [1, 2, 3][:1];\n {[1]}\n db> select [1, 2, 3][:-2];\n {[1]}\n\n Referencing an array slice beyond the array boundaries will result in an\n empty array (unlike a direct reference to a specific index). Slicing with\n a lower bound less than the minimum index or a upper bound greater than\n the maximum index are functionally equivalent to not specifying those\n bounds for your slice:\n\n .. code-block:: edgeql-repl\n\n db> select [1, 2, 3][1:20];\n {[2, 3]}\n db> select [1, 2, 3][10:20];\n {[]}\n\n\n---------\n\n\n.. eql:operator:: arrayplus: array ++ array -> array\n\n .. index:: concatenate, join, add\n .. api-index:: Ā§array Ā§++Ā§ arrayĀ§\n\n Concatenates two arrays of the same type into one.\n\n .. code-block:: edgeql-repl\n\n db> select [1, 2, 3] ++ [99, 98];\n {[1, 2, 3, 99, 98]}\n\n\n----------\n\n\n.. eql:function:: std::array_agg(s: set of anytype) -> array\n\n .. index:: aggregate\n\n Returns an array made from all of the input set elements.\n\n The ordering of the input set will be preserved if specified:\n\n .. code-block:: edgeql-repl\n\n db> select array_agg({2, 3, 5});\n {[2, 3, 5]}\n\n db> select array_agg(User.name order by User.name);\n {['Alice', 'Bob', 'Joe', 'Sam']}\n\n\n----------\n\n\n.. eql:function:: std::array_get(array: array, \\\n index: int64, \\\n named only default: anytype = {} \\\n ) -> optional anytype\n\n Returns the element of a given *array* at the specified *index*.\n\n If the index is out of the array's bounds, the *default* argument or\n ``{}`` (empty set) will be returned.\n\n This works the same as the :eql:op:`array indexing operator `,\n except that if the index is out of bounds, an empty set\n of the array element's type is returned instead of raising an exception:\n\n .. code-block:: edgeql-repl\n\n db> select array_get([2, 3, 5], 1);\n {3}\n db> select array_get([2, 3, 5], 100);\n {}\n db> select array_get([2, 3, 5], 100, default := 42);\n {42}\n\n\n----------\n\n\n.. eql:function:: std::array_unpack(array: array) -> set of anytype\n\n Returns the elements of an array as a set.\n\n .. note::\n\n The ordering of the returned set is not guaranteed.\n However, if it is wrapped in a call to :eql:func:`enumerate`,\n the assigned indexes are guaranteed to match the array.\n\n .. code-block:: edgeql-repl\n\n db> select array_unpack([2, 3, 5]);\n {3, 2, 5}\n\n db> select enumerate(array_unpack([2, 3, 5]));\n {(1, 3), (0, 2), (2, 5)}\n\n\n----------\n\n\n.. eql:function:: std::array_join(array: array, delimiter: str) -> str\n std::array_join(array: array, \\\n delimiter: bytes) -> bytes\n\n .. index:: array_to_string, implode\n\n Renders an array to a string or byte-string.\n\n Join a string array into a single string using a specified *delimiter*:\n\n .. code-block:: edgeql-repl\n\n db> select array_join(['one', 'two', 'three'], ', ');\n {'one, two, three'}\n\n Similarly, an array of :eql:type:`bytes` can be joined as a single value\n using a specified *delimiter*:\n\n .. code-block:: edgeql-repl\n\n db> select array_join([b'\\x01', b'\\x02', b'\\x03'], b'\\xff');\n {b'\\x01\\xff\\x02\\xff\\x03'}\n\n\n----------\n\n\n.. eql:function:: std::array_fill(val: anytype, n: int64) -> array\n\n Returns an array of the specified size, filled with the provided value.\n\n Create an array of size *n* where every element has the value *val*.\n\n .. code-block:: edgeql-repl\n\n db> select array_fill(0, 5);\n {[0, 0, 0, 0, 0]}\n db> select array_fill('n/a', 3);\n {['n/a', 'n/a', 'n/a']}\n\n\n----------\n\n\n.. eql:function:: std::array_replace(array: array, \\\n old: anytype, \\\n new: anytype) \\\n -> array\n\n Returns an array with all occurrences of one value replaced by another.\n\n Return an array where every *old* value is replaced with *new*.\n\n .. code-block:: edgeql-repl\n\n db> select array_replace([1, 1, 2, 3, 5], 1, 99);\n {[99, 99, 2, 3, 5]}\n db> select array_replace(['h', 'e', 'l', 'l', 'o'], 'l', 'L');\n {['h', 'e', 'L', 'L', 'o']}\n\n\n----------\n\n\n.. eql:function:: std::array_set(array: array, \\\n idx: int64, \\\n val: anytype) \\\n -> array\n\n .. versionadded:: 6.0\n\n Returns an array with an value at a specific index replaced by another.\n\n Return an array where the value at the index indicated by *idx* is\n replaced with *val*.\n\n .. code-block:: edgeql-repl\n\n db> select array_set(['hello', 'world'], 0, 'goodbye');\n {['goodbye', 'world']}\n db> select array_set([1, 1, 2, 3], 1, 99);\n {[1, 99, 2, 3]}\n\n\n----------\n\n\n.. eql:function:: std::array_insert(array: array, \\\n idx: int64, \\\n val: anytype) \\\n -> array\n\n .. versionadded:: 6.0\n\n Returns an array with an value inserted at a specific.\n\n Return an array where the value *val* is inserted at the index indicated by *idx*.\n\n .. code-block:: edgeql-repl\n\n db> select array_insert(['the', 'brown', 'fox'], 1, 'quick');\n {['the', 'quick', 'brown', 'fox']}\n db> select array_insert([1, 1, 2, 3], 1, 99);\n {[1, 99, 1, 2, 3]}\n" }, { "path": "docs/reference/stdlib/bool.rst", "content": ".. _ref_std_logical:\n\n\n========\nBooleans\n========\n\n:edb-alt-title: Boolean Functions and Operators\n\n\n.. list-table::\n :class: funcoptable\n\n * - :eql:type:`bool`\n - Boolean type\n\n * - :eql:op:`bool or bool `\n - :eql:op-desc:`or`\n\n * - :eql:op:`bool and bool `\n - :eql:op-desc:`and`\n\n * - :eql:op:`not bool `\n - :eql:op-desc:`not`\n\n * - :eql:op:`= ` :eql:op:`\\!= ` :eql:op:`?= `\n :eql:op:`?!= ` :eql:op:`\\< ` :eql:op:`\\> `\n :eql:op:`\\<= ` :eql:op:`\\>= `\n - Comparison operators\n\n * - :eql:func:`all`\n - :eql:func-desc:`all`\n\n * - :eql:func:`any`\n - :eql:func-desc:`any`\n\n * - :eql:func:`assert`\n - :eql:func-desc:`assert`\n\n----------\n\n\n.. eql:type:: std::bool\n\n A boolean type of either ``true`` or ``false``.\n\n EdgeQL has case-insensitive keywords and that includes the boolean\n literals:\n\n .. code-block:: edgeql-repl\n\n db> select (True, true, TRUE);\n {(true, true, true)}\n db> select (False, false, FALSE);\n {(false, false, false)}\n\n These basic operators will always result in a boolean type value (although,\n for some of them, that value may be the empty set if an operand is the\n empty set):\n\n - :eql:op:`= `\n - :eql:op:`\\!= `\n - :eql:op:`?= `\n - :eql:op:`?!= `\n - :eql:op:`in`\n - :eql:op:`not in `\n - :eql:op:`\\< `\n - :eql:op:`\\> `\n - :eql:op:`\\<= `\n - :eql:op:`\\>= `\n - :eql:op:`like`\n - :eql:op:`ilike`\n\n These operators will result in a boolean type value even if the right\n operand is the empty set:\n\n - :eql:op:`in`\n - :eql:op:`not in `\n\n These operators will always result in a boolean ``true`` or ``false``\n value, even if either operand is the empty set:\n\n - :eql:op:`?= `\n - :eql:op:`?!= `\n\n These operators will produce the empty set if either operand is the empty\n set:\n\n - :eql:op:`= `\n - :eql:op:`\\!= `\n - :eql:op:`\\< `\n - :eql:op:`\\> `\n - :eql:op:`\\<= `\n - :eql:op:`\\>= `\n - :eql:op:`like`\n - :eql:op:`ilike`\n\n If you need to use these operators and it's possible one or both operands\n will be the empty set, you can ensure a ``bool`` product by\n :eql:op:`coalescing `. With ``=`` and ``!=``, you can use their\n respective dedicated coalescing operators, ``?=`` and ``?!=``. See each\n individual operator for an example.\n\n Some boolean operator examples:\n\n .. code-block:: edgeql-repl\n\n db> select true and 2 < 3;\n {true}\n db> select '!' IN {'hello', 'world'};\n {false}\n\n It's possible to get a boolean by casting a :eql:type:`str` or\n :eql:type:`json` value into it:\n\n .. code-block:: edgeql-repl\n\n db> select ('true');\n {true}\n db> select to_json('false');\n {false}\n\n :ref:`Filter clauses ` must\n always evaluate to a boolean:\n\n .. code-block:: edgeql\n\n select User\n filter .name ilike 'alice';\n\n\n----------\n\n\n.. eql:operator:: or: bool or bool -> bool\n\n .. api-index:: or\n\n Evaluates ``true`` if either boolean is ``true``.\n\n .. code-block:: edgeql-repl\n\n db> select false or true;\n {true}\n\n .. warning::\n\n When either operand in an ``or`` is an empty set, the result will not\n be a ``bool`` but instead an empty set.\n\n .. code-block:: edgeql-repl\n\n db> select true or {};\n {}\n\n If one of the operands in an ``or`` operation could be an empty set,\n you may want to use the :eql:op:`coalesce` operator (``??``) on that\n side to ensure you will still get a ``bool`` result.\n\n .. code-block:: edgeql-repl\n\n db> select true or ({} ?? false);\n {true}\n\n\n----------\n\n\n.. eql:operator:: and: bool and bool -> bool\n\n .. api-index:: and\n\n Evaluates ``true`` if both booleans are ``true``.\n\n .. code-block:: edgeql-repl\n\n db> select false and true;\n {false}\n\n .. warning::\n\n When either operand in an ``and`` is an empty set, the result will not\n be a ``bool`` but instead an empty set.\n\n .. code-block:: edgeql-repl\n\n db> select true and {};\n {}\n\n If one of the operands in an ``and`` operation could be an empty set,\n you may want to use the :eql:op:`coalesce` operator (``??``) on that\n side to ensure you will still get a ``bool`` result.\n\n .. code-block:: edgeql-repl\n\n db> select true and ({} ?? false);\n {false}\n\n\n----------\n\n\n.. eql:operator:: not: not bool -> bool\n\n .. api-index:: not\n\n Logically negates a given boolean value.\n\n .. code-block:: edgeql-repl\n\n db> select not false;\n {true}\n\n .. warning::\n\n When the operand in a ``not`` is an empty set, the result will not be a\n ``bool`` but instead an empty set.\n\n .. code-block:: edgeql-repl\n\n db> select not {};\n {}\n\n If the operand in a ``not`` operation could be an empty set, you may\n want to use the :eql:op:`coalesce` operator (``??``) on that side to\n ensure you will still get a ``bool`` result.\n\n .. code-block:: edgeql-repl\n\n db> select not ({} ?? false);\n {true}\n\n\n----------\n\n\nThe ``and`` and ``or`` operators are commutative.\n\nThe truth tables are as follows:\n\n+-------+-------+---------------+--------------+--------------+\n| a | b | a ``and`` b | a ``or`` b | ``not`` a |\n+=======+=======+===============+==============+==============+\n| true | true | true | true | false |\n+-------+-------+---------------+--------------+--------------+\n| true | false | false | true | false |\n+-------+-------+---------------+--------------+--------------+\n| false | true | false | true | true |\n+-------+-------+---------------+--------------+--------------+\n| false | false | false | false | true |\n+-------+-------+---------------+--------------+--------------+\n\n\n----------\n\nThe operators ``and``/``or`` and the functions :eql:func:`all`/:eql:func:`any`\ndiffer in the way they handle an empty set (``{}``). Both ``and`` and ``or``\noperators apply to the cross-product of their operands. If either operand is\nan empty set, the result will also be an empty set. For example:\n\n.. code-block:: edgeql-repl\n\n db> select {true, false} and {};\n {}\n db> select true and {};\n {}\n\nOperating on an empty set with :eql:func:`all`/:eql:func:`any` does *not*\nreturn an empty set:\n\n.. code-block:: edgeql-repl\n\n db> select all({});\n {true}\n db> select any({});\n {false}\n\n:eql:func:`all` returns ``true`` because the empty set contains no false\nvalues.\n\n:eql:func:`any` returns ``false`` because the empty set contains no\ntrue values.\n\nThe :eql:func:`all` and :eql:func:`any` functions are generalized to apply to\nsets of values, including ``{}``. Thus they have the following truth\ntable:\n\n+-------+-------+-----------------+-----------------+\n| a | b | ``all({a, b})`` | ``any({a, b})`` |\n+=======+=======+=================+=================+\n| true | true | true | true |\n+-------+-------+-----------------+-----------------+\n| true | false | false | true |\n+-------+-------+-----------------+-----------------+\n| {} | true | true | true |\n+-------+-------+-----------------+-----------------+\n| {} | false | false | false |\n+-------+-------+-----------------+-----------------+\n| false | true | false | true |\n+-------+-------+-----------------+-----------------+\n| false | false | false | false |\n+-------+-------+-----------------+-----------------+\n| true | {} | true | true |\n+-------+-------+-----------------+-----------------+\n| false | {} | false | false |\n+-------+-------+-----------------+-----------------+\n| {} | {} | true | false |\n+-------+-------+-----------------+-----------------+\n\nSince :eql:func:`all` and :eql:func:`any` apply to sets as a whole,\nmissing values (represented by ``{}``) are just that - missing. They\ndon't affect the overall result.\n\nTo understand the last line in the above truth table it's useful to\nremember that ``all({a, b}) = all(a) and all(b)`` and ``any({a, b}) =\nany(a) or any(b)``.\n\nFor more customized handling of ``{}``, use the :eql:op:`?? `\noperator.\n\n\n----------\n\n\n.. eql:function:: std::assert( \\\n input: bool, \\\n named only message: optional str = {} \\\n ) -> bool\n\n Checks that the input bool is ``true``.\n\n If the input bool is ``false``, ``assert`` raises a\n ``QueryAssertionError``. Otherwise, this function returns ``true``.\n\n .. code-block:: edgeql-repl\n\n db> select assert(true);\n {true}\n\n db> select assert(false);\n gel error: QueryAssertionError: assertion failed\n\n db> select assert(false, message := 'value is not true');\n gel error: QueryAssertionError: value is not true\n\n ``assert`` can be used in triggers to create more powerful constraints. In\n this schema, the ``Person`` type has both ``friends`` and ``enemies``\n links. You may not want a ``Person`` to be both a friend and an enemy of\n the same ``Person``. ``assert`` can be used inside a trigger to easily\n prohibit this.\n\n .. code-block:: sdl\n\n type Person {\n required name: str;\n multi friends: Person;\n multi enemies: Person;\n\n trigger prohibit_frenemies after insert, update for each do (\n assert(\n not exists (__new__.friends intersect __new__.enemies),\n message := \"Invalid frenemies\",\n )\n )\n }\n\n With this trigger in place, it is impossible to link the same ``Person`` as\n both a friend and an enemy of any other person.\n\n .. code-block:: edgeql-repl\n\n db> insert Person {name := 'Quincey Morris'};\n {default::Person {id: e4a55480-d2de-11ed-93bd-9f4224fc73af}}\n db> insert Person {name := 'Dracula'};\n {default::Person {id: e7f2cff0-d2de-11ed-93bd-279780478afb}}\n db> update Person\n ... filter .name = 'Quincey Morris'\n ... set {\n ... enemies := (\n ... select detached Person filter .name = 'Dracula'\n ... )\n ... };\n {default::Person {id: e4a55480-d2de-11ed-93bd-9f4224fc73af}}\n db> update Person\n ... filter .name = 'Quincey Morris'\n ... set {\n ... friends := (\n ... select detached Person filter .name = 'Dracula'\n ... )\n ... };\n gel error: GelError: Invalid frenemies\n\n In the following examples, the ``size`` properties of the ``File`` objects\n are ``1024``, ``1024``, and ``131,072``.\n\n .. code-block:: edgeql-repl\n\n db> for obj in (select File)\n ... union (assert(obj.size <= 128*1024, message := 'file too big'));\n {true, true, true}\n\n db> for obj in (select File)\n ... union (assert(obj.size <= 64*1024, message := 'file too big'));\n gel error: QueryAssertionError: file too big\n\n You may call ``assert`` in the ``order by`` clause of your ``select``\n statement. This will ensure it is called only on objects that pass your\n filter.\n\n .. code-block:: edgeql-repl\n\n db> select File { name, size }\n ... order by assert(.size <= 128*1024, message := \"file too big\");\n {\n default::File {name: 'File 2', size: 1024},\n default::File {name: 'Asdf 3', size: 1024},\n default::File {name: 'File 1', size: 131072},\n }\n\n db> select File { name, size }\n ... order by assert(.size <= 64*1024, message := \"file too big\");\n gel error: QueryAssertionError: file too big\n\n db> select File { name, size }\n ... filter .size <= 64*1024\n ... order by assert(.size <= 64*1024, message := \"file too big\");\n {\n default::File {name: 'File 2', size: 1024},\n default::File {name: 'Asdf 3', size: 1024}\n }\n" }, { "path": "docs/reference/stdlib/bytes.rst", "content": ".. _ref_std_bytes:\n\n=====\nBytes\n=====\n\n:edb-alt-title: Bytes Functions and Operators\n\n.. list-table::\n :class: funcoptable\n\n * - :eql:type:`bytes`\n - Byte sequence\n\n * - :eql:type:`Endian`\n - An enum for indicating integer value encoding.\n\n * - :eql:op:`bytes[i] `\n - :eql:op-desc:`bytesidx`\n\n * - :eql:op:`bytes[from:to] `\n - :eql:op-desc:`bytesslice`\n\n * - :eql:op:`bytes ++ bytes `\n - :eql:op-desc:`bytesplus`\n\n * - :eql:op:`= ` :eql:op:`\\!= ` :eql:op:`?= `\n :eql:op:`?!= ` :eql:op:`\\< ` :eql:op:`\\> `\n :eql:op:`\\<= ` :eql:op:`\\>= `\n - Comparison operators\n\n * - :eql:func:`len`\n - Returns the number of bytes.\n\n * - :eql:func:`contains`\n - Checks if the byte sequence contains a given subsequence.\n\n * - :eql:func:`find`\n - Finds the index of the first occurrence of a subsequence.\n\n * - :eql:func:`to_bytes`\n - :eql:func-desc:`to_bytes`\n\n * - :eql:func:`to_str`\n - :eql:func-desc:`to_str`\n\n * - :eql:func:`to_int16`\n - :eql:func-desc:`to_int16`\n\n * - :eql:func:`to_int32`\n - :eql:func-desc:`to_int32`\n\n * - :eql:func:`to_int64`\n - :eql:func-desc:`to_int64`\n\n * - :eql:func:`to_uuid`\n - :eql:func-desc:`to_uuid`\n\n * - :eql:func:`bytes_get_bit`\n - :eql:func-desc:`bytes_get_bit`\n\n * - :eql:func:`bit_count`\n - :eql:func-desc:`bit_count`\n\n * - :eql:func:`enc::base64_encode`\n - :eql:func-desc:`enc::base64_encode`\n\n * - :eql:func:`enc::base64_decode`\n - :eql:func-desc:`enc::base64_decode`\n\n----------\n\n\n.. eql:type:: std::bytes\n\n A sequence of bytes representing raw data.\n\n Bytes can be represented as a literal using this syntax: ``b''``.\n\n .. code-block:: edgeql-repl\n\n db> select b'Hello, world';\n {b'Hello, world'}\n db> select b'Hello,\\x20world\\x01';\n {b'Hello, world\\x01'}\n\n There are also some :ref:`generic `\n functions that can operate on bytes:\n\n .. code-block:: edgeql-repl\n\n db> select contains(b'qwerty', b'42');\n {false}\n\n Bytes are rendered as base64-encoded strings in JSON. When you cast a\n ``bytes`` value into JSON, that's what you'll get. In order to\n :eql:op:`cast ` a :eql:type:`json` value into bytes, it must be a\n base64-encoded string.\n\n .. code-block:: edgeql-repl\n\n db> select b'Hello Gel!';\n {\"\\\"SGVsbG8gRWRnZURCIQ==\\\"\"}\n db> select to_json(\"\\\"SGVsbG8gRWRnZURCIQ==\\\"\");\n {b'Hello Gel!'}\n\n\n----------\n\n\n.. eql:type:: std::Endian\n\n .. versionadded:: 5.0\n\n An enum for indicating integer value encoding.\n\n This enum is used by the :eql:func:`to_int16`, :eql:func:`to_int32`,\n :eql:func:`to_int64` and the :eql:func:`to_bytes` converters working with\n :eql:type:`bytes` and integers.\n\n ``Endian.Big`` stands for big-endian encoding going from most significant\n byte to least. ``Endian.Little`` stands for little-endian encoding going\n from least to most significant byte.\n\n .. code-block:: edgeql-repl\n\n db> select to_bytes(16908295, Endian.Big);\n {b'\\x01\\x02\\x00\\x07'}\n db> select to_int32(b'\\x01\\x02\\x00\\x07', Endian.Big);\n {16908295}\n db> select to_bytes(16908295, Endian.Little);\n {b'\\x07\\x00\\x02\\x01'}\n db> select to_int32(b'\\x07\\x00\\x02\\x01', Endian.Little);\n {16908295}\n\n\n----------\n\n\n.. eql:operator:: bytesidx: bytes [ int64 ] -> bytes\n\n .. api-index:: Ā§bytesĀ§[Ā§intĀ§]\n\n Accesses a byte at a given index.\n\n Examples:\n\n .. code-block:: edgeql-repl\n\n db> select b'binary \\x01\\x02\\x03\\x04 ftw!'[2];\n {b'n'}\n db> select b'binary \\x01\\x02\\x03\\x04 ftw!'[8];\n {b'\\x02'}\n\n\n----------\n\n\n.. eql:operator:: bytesslice: bytes [ int64 : int64 ] -> bytes\n\n .. api-index:: Ā§bytesĀ§[Ā§intĀ§:Ā§intĀ§]\n\n Produces a bytes sub-sequence from an existing bytes value.\n\n Examples:\n\n .. code-block:: edgeql-repl\n\n db> select b'\\x01\\x02\\x03\\x04 ftw!'[2:-1];\n {b'\\x03\\x04 ftw'}\n db> select b'some bytes'[2:-3];\n {b'me by'}\n\n\n---------\n\n\n.. eql:operator:: bytesplus: bytes ++ bytes ->\n\n .. index:: join, add\n .. api-index:: Ā§bytes Ā§++Ā§ bytesĀ§\n\n Concatenates two bytes values into one.\n\n .. code-block:: edgeql-repl\n\n db> select b'\\x01\\x02' ++ b'\\x03\\x04';\n {b'\\x01\\x02\\x03\\x04'}\n\n\n---------\n\n.. TODO: Function signatures except the first need to be revealed only for v5+\n\n.. eql:function:: std::to_bytes(s: str) -> bytes\n std::to_bytes(j: json) -> bytes\n std::to_bytes(val: int16, endian: Endian) -> bytes\n std::to_bytes(val: int32, endian: Endian) -> bytes\n std::to_bytes(val: int64, endian: Endian) -> bytes\n std::to_bytes(val: uuid) -> bytes\n\n .. versionadded:: 4.0\n\n .. index:: encode, stringencoder\n\n Converts a given value into binary representation as :eql:type:`bytes`.\n\n The strings get converted using UTF-8 encoding:\n\n .. code-block:: edgeql-repl\n\n db> select to_bytes('ćƒ†ć‚­ć‚¹ćƒˆ');\n {b'\\xe3\\x83\\x86\\xe3\\x82\\xad\\xe3\\x82\\xb9\\xe3\\x83\\x88'}\n\n The json values get converted as strings using UTF-8 encoding:\n\n .. code-block:: edgeql-repl\n\n db> select to_bytes(to_json('{\"a\": 1}'));\n {b'{\"a\": 1}'}\n\n The integer values can be encoded as big-endian (most significant bit\n comes first) byte strings:\n\n .. code-block:: edgeql-repl\n\n db> select to_bytes(31, Endian.Big);\n {b'\\x00\\x1f'}\n db> select to_bytes(31, Endian.Big);\n {b'\\x00\\x00\\x00\\x1f'}\n db> select to_bytes(123456789123456789, Endian.Big);\n {b'\\x01\\xb6\\x9bK\\xac\\xd0_\\x15'}\n\n .. note::\n\n Due to underlying implementation details using big-endian encoding\n results in slightly faster performance of ``to_bytes`` when converting\n integers.\n\n The UUID values are converted to the underlying string of 16 bytes:\n\n .. code-block:: edgeql-repl\n\n db> select to_bytes('1d70c86e-cc92-11ee-b4c7-a7aa0a34e2ae');\n {b'\\x1dp\\xc8n\\xcc\\x92\\x11\\xee\\xb4\\xc7\\xa7\\xaa\\n4\\xe2\\xae'}\n\n To perform the reverse conversion there are corresponding functions:\n :eql:func:`to_str`, :eql:func:`to_int16`, :eql:func:`to_int32`,\n :eql:func:`to_int64`, :eql:func:`to_uuid`.\n\n\n---------\n\n.. eql:function:: std::bytes_get_bit(bytes: bytes, nth: int64) -> int64\n\n Returns the specified bit of the :eql:type:`bytes` value.\n\n When looking for the *nth* bit, this function will enumerate bits from\n least to most significant in each byte.\n\n .. code-block:: edgeql-repl\n\n db> for n in {0, 1, 2, 3, 4, 5, 6, 7,\n ... 8, 9, 10, 11, 12, 13 ,14, 15}\n ... union bytes_get_bit(b'ab', n);\n {1, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0}\n\n\n---------\n\n\n.. eql:function:: enc::base64_encode(b: bytes) -> str\n\n .. versionadded:: 4.0\n\n Returns a Base64-encoded :eql:type:`str` of the :eql:type:`bytes` value.\n\n .. code-block:: edgeql-repl\n\n db> select enc::base64_encode(b'hello');\n {'aGVsbG8='}\n\n---------\n\n.. eql:function:: enc::base64_decode(s: str) -> bytes\n\n .. versionadded:: 4.0\n\n Returns the :eql:type:`bytes` of a Base64-encoded :eql:type:`str`.\n\n Returns an InvalidValueError if input is not valid Base64.\n\n .. code-block:: edgeql-repl\n\n db> select enc::base64_decode('aGVsbG8=');\n {b'hello'}\n db> select enc::base64_decode('aGVsbG8');\n gel error: InvalidValueError: invalid base64 end sequence\n" }, { "path": "docs/reference/stdlib/cfg.rst", "content": ".. _ref_std_cfg:\n\n======\nConfig\n======\n\nThe ``cfg`` module contains a set of types and scalars used for configuring\n|Gel|.\n\n\n.. list-table::\n :class: funcoptable\n\n * - **Type**\n - **Description**\n * - :eql:type:`cfg::AbstractConfig`\n - The abstract base type for all configuration objects. The properties\n of this type define the set of configuruation settings supported by\n Gel.\n * - :eql:type:`cfg::Config`\n - The main configuration object. The properties of this object reflect\n the overall configuration setting from instance level all the way to\n session level.\n * - :eql:type:`cfg::DatabaseConfig`\n - The database configuration object. It reflects all the applicable\n configuration at the Gel database level.\n * - :eql:type:`cfg::BranchConfig`\n - The database branch configuration object. It reflects all the applicable\n configuration at the Gel branch level.\n * - :eql:type:`cfg::InstanceConfig`\n - The instance configuration object.\n * - :eql:type:`cfg::ExtensionConfig`\n - The abstract base type for all extension configuration objects. Each\n extension can define the necessary configuration settings by extending\n this type and adding the extension-specific properties.\n * - :eql:type:`cfg::Auth`\n - An object type representing an authentication profile.\n * - :eql:type:`cfg::ConnectionTransport`\n - An enum type representing the different protocols that Gel speaks.\n * - :eql:type:`cfg::AuthMethod`\n - An abstract object type representing a method of authentication\n * - :eql:type:`cfg::Trust`\n - A subclass of ``AuthMethod`` indicating an \"always trust\" policy (no\n authentication).\n * - :eql:type:`cfg::SCRAM`\n - A subclass of ``AuthMethod`` indicating password-based authentication.\n * - :eql:type:`cfg::Password`\n - A subclass of ``AuthMethod`` indicating basic password-based\n authentication.\n * - :eql:type:`cfg::JWT`\n - A subclass of ``AuthMethod`` indicating token-based authentication.\n * - :eql:type:`cfg::memory`\n - A scalar type for storing a quantity of memory storage.\n\n\n.. eql:type:: cfg::AbstractConfig\n\n An abstract type representing the configuration of an instance or database.\n\n The properties of this object type represent the set of configuration\n options supported by Gel (listed above).\n\n\n----------\n\n\n.. eql:type:: cfg::Config\n\n The main configuration object type.\n\n This type will have only one object instance. The ``cfg::Config`` object\n represents the sum total of the current Gel configuration. It reflects\n the result of applying instance, branch, and session level configuration.\n Examining this object is the recommended way of determining the current\n configuration.\n\n Here's an example of checking and disabling :ref:`access policies\n `:\n\n .. code-block:: edgeql-repl\n\n db> select cfg::Config.apply_access_policies;\n {true}\n db> configure session set apply_access_policies := false;\n OK: CONFIGURE SESSION\n db> select cfg::Config.apply_access_policies;\n {false}\n\n\n----------\n\n\n.. eql:type:: cfg::BranchConfig\n\n .. versionadded:: 5.0\n\n The branch-level configuration object type.\n\n This type will have only one object instance. The ``cfg::BranchConfig``\n object represents the state of the branch and instance-level Gel\n configuration.\n\n For overall configuration state please refer to the :eql:type:`cfg::Config`\n instead.\n\n\n----------\n\n\n.. eql:type:: cfg::InstanceConfig\n\n The instance-level configuration object type.\n\n This type will have only one object instance. The ``cfg::InstanceConfig``\n object represents the state of only instance-level Gel configuration.\n\n For overall configuraiton state please refer to the :eql:type:`cfg::Config`\n instead.\n\n\n----------\n\n\n.. eql:type:: cfg::ExtensionConfig\n\n .. versionadded:: 5.0\n\n An abstract type representing extension configuration.\n\n Every extension is expected to define its own extension-specific config\n object type extending ``cfg::ExtensionConfig``. Any necessary extension\n configuration setting should be represented as properties of this concrete\n config type.\n\n Up to three instances of the extension-specific config type will be created,\n each of them with a ``required single link cfg`` to the\n :eql:type:`cfg::Config`, :eql:type:`cfg::DatabaseConfig`, or\n :eql:type:`cfg::InstanceConfig` object depending on the configuration level.\n The :eql:type:`cfg::AbstractConfig` exposes a corresponding computed\n multi-backlink called ``extensions``.\n\n For example, :ref:`ext::pgvector ` extension exposes\n ``probes`` as a configurable parameter via ``ext::pgvector::Config`` object:\n\n .. code-block:: edgeql-repl\n\n db> configure session\n ... set ext::pgvector::Config::probes := 5;\n OK: CONFIGURE SESSION\n db> select cfg::Config.extensions[is ext::pgvector::Config]{*};\n {\n ext::pgvector::Config {\n id: 12b5c70f-0bb8-508a-845f-ca3d41103b6f,\n probes: 5,\n ef_search: 40,\n },\n }\n\n\n----------\n\n\n.. eql:type:: cfg::Auth\n\n An object type designed to specify a client authentication profile.\n\n .. code-block:: edgeql-repl\n\n db> configure instance insert\n ... Auth {priority := 0, method := (insert Trust)};\n OK: CONFIGURE INSTANCE\n\n Below are the properties of the ``Auth`` class.\n\n :eql:synopsis:`priority -> int64`\n The priority of the authentication rule. The lower this number,\n the higher the priority.\n\n :eql:synopsis:`user -> multi str`\n The name(s) of the database role(s) this rule applies to. If set to\n ``'*'``, then it applies to all roles.\n\n :eql:synopsis:`method -> cfg::AuthMethod`\n The name of the authentication method type. Expects an instance of\n :eql:type:`cfg::AuthMethod`; Valid values are:\n ``Trust`` for no authentication and ``SCRAM`` for SCRAM-SHA-256\n password authentication.\n\n :eql:synopsis:`comment -> optional str`\n An optional comment for the authentication rule.\n\n---------\n\n.. eql:type:: cfg::ConnectionTransport\n\n An enum listing the various protocols that Gel can speak.\n\n Possible values are:\n\n.. list-table::\n :class: funcoptable\n\n * - **Value**\n - **Description**\n * - ``cfg::ConnectionTransport.TCP``\n - Gel binary protocol\n * - ``cfg::ConnectionTransport.TCP_PG``\n - Postgres protocol for the\n :ref:`SQL query mode `\n * - ``cfg::ConnectionTransport.HTTP``\n - Gel binary protocol\n :ref:`tunneled over HTTP `\n * - ``cfg::ConnectionTransport.SIMPLE_HTTP``\n - :ref:`EdgeQL over HTTP `\n and :ref:`GraphQL ` endpoints\n\n---------\n\n.. eql:type:: cfg::AuthMethod\n\n An abstract object class that represents an authentication method.\n\n It currently has four concrete subclasses, each of which represent an\n available authentication method: :eql:type:`cfg::SCRAM`,\n :eql:type:`cfg::JWT`, :eql:type:`cfg::Password`, and\n :eql:type:`cfg::Trust`.\n\n :eql:synopsis:`transports -> multi cfg::ConnectionTransport`\n Which connection transports this method applies to.\n The subclasses have their own defaults for this.\n\n-------\n\n.. eql:type:: cfg::Trust\n\n The ``cfg::Trust`` indicates an \"always-trust\" policy.\n\n When active, it disables password-based authentication.\n\n .. code-block:: edgeql-repl\n\n db> configure instance insert\n ... Auth {priority := 0, method := (insert Trust)};\n OK: CONFIGURE INSTANCE\n\n-------\n\n.. eql:type:: cfg::SCRAM\n\n ``cfg::SCRAM`` indicates password-based authentication.\n\n It uses a challenge-response scheme to avoid transmitting the\n password directly. This policy is implemented via ``SCRAM-SHA-256``\n\n It is available for the ``TCP``, ``TCP_PG``, and ``HTTP`` transports\n and is the default for ``TCP`` and ``TCP_PG``.\n\n .. code-block:: edgeql-repl\n\n db> configure instance insert\n ... Auth {priority := 0, method := (insert SCRAM)};\n OK: CONFIGURE INSTANCE\n\n-------\n\n.. eql:type:: cfg::JWT\n\n ``cfg::JWT`` uses a JWT signed by the server to authenticate.\n\n It is available for the ``TCP``, ``HTTP``, and ``HTTP_SIMPLE`` transports\n and is the default for ``HTTP``.\n\n\n-------\n\n.. eql:type:: cfg::Password\n\n ``cfg::Password`` indicates simple password-based authentication.\n\n Unlike :eql:type:`cfg::SCRAM`, this policy transmits the password\n over the (encrypted) channel. It is implemened using HTTP Basic\n Authentication over TLS.\n\n This policy is available only for the ``SIMPLE_HTTP`` transport, where it is\n the default.\n\n\n-------\n\n.. eql:type:: cfg::memory\n\n A scalar type representing a quantity of memory storage.\n\n As with ``uuid``, ``datetime``, and several other types, ``cfg::memory``\n values are declared by casting from an appropriately formatted string.\n\n .. code-block:: edgeql-repl\n\n db> select '1B'; # 1 byte\n {'1B'}\n db> select '5KiB'; # 5 kibibytes\n {'5KiB'}\n db> select '128MiB'; # 128 mebibytes\n {'128MiB'}\n\n The numerical component of the value must be a non-negative integer; the\n units must be one of ``B|KiB|MiB|GiB|TiB|PiB``. We're using the explicit\n ``KiB`` unit notation (1024 bytes) instead of ``kB`` (which is ambiguous,\n and may mean 1000 or 1024 bytes).\n" }, { "path": "docs/reference/stdlib/constraint_table.rst", "content": ".. list-table::\n\n * - :eql:constraint:`exclusive`\n - Enforce uniqueness (disallow duplicate values)\n\n * - :eql:constraint:`expression`\n - Custom constraint expression (followed by keyword ``on``)\n\n * - :eql:constraint:`one_of`\n - A list of allowable values\n\n * - :eql:constraint:`max_value`\n - Maximum value numerically/lexicographically\n\n * - :eql:constraint:`max_ex_value`\n - Maximum value numerically/lexicographically (exclusive range)\n\n * - :eql:constraint:`min_value`\n - Minimum value numerically/lexicographically\n\n * - :eql:constraint:`min_ex_value`\n - Minimum value numerically/lexicographically (exclusive range)\n\n * - :eql:constraint:`max_len_value`\n - Maximum length (``str`` only)\n\n * - :eql:constraint:`min_len_value`\n - Minimum length (``str`` only)\n\n * - :eql:constraint:`regexp`\n - Regex constraint (``str`` only)\n\n" }, { "path": "docs/reference/stdlib/constraints.rst", "content": ".. _ref_std_constraints:\n\n===========\nConstraints\n===========\n\n.. include:: constraint_table.rst\n\n\n.. eql:constraint:: std::expression on (expr)\n\n A constraint based on an arbitrary expression returning a boolean.\n\n The ``expression`` constraint may be used as in this example to create a\n custom scalar type:\n\n .. code-block:: sdl\n\n scalar type StartsWithA extending str {\n constraint expression on (__subject__[0] = 'A');\n }\n\n Example of using an ``expression`` constraint based on two\n object properties to restrict maximum magnitude for a vector:\n\n .. code-block:: sdl\n\n type Vector {\n required x: float64;\n required y: float64;\n constraint expression on (\n __subject__.x^2 + __subject__.y^2 < 25\n );\n }\n\n.. eql:constraint:: std::one_of(variadic members: anytype)\n\n Specifies a list of allowed values.\n\n Example:\n\n .. code-block:: sdl\n\n scalar type Status extending str {\n constraint one_of ('Open', 'Closed', 'Merged');\n }\n\n.. eql:constraint:: std::max_value(max: anytype)\n\n Specifies the maximum allowed value.\n\n Example:\n\n .. code-block:: sdl\n\n scalar type Max100 extending int64 {\n constraint max_value(100);\n }\n\n.. eql:constraint:: std::max_ex_value(max: anytype)\n\n Specifies a non-inclusive upper bound for the value.\n\n Example:\n\n .. code-block:: sdl\n\n scalar type Under100 extending int64 {\n constraint max_ex_value(100);\n }\n\n In this example, in contrast to the ``max_value`` constraint, a value\n of the ``Under100`` type cannot be ``100`` since the valid range of\n ``max_ex_value`` does not include the value specified in the constraint.\n\n.. eql:constraint:: std::max_len_value(max: int64)\n\n Specifies the maximum allowed length of a value.\n\n Example:\n\n .. code-block:: sdl\n\n scalar type Username extending str {\n constraint max_len_value(30);\n }\n\n.. eql:constraint:: std::min_value(min: anytype)\n\n Specifies the minimum allowed value.\n\n Example:\n\n .. code-block:: sdl\n\n scalar type NonNegative extending int64 {\n constraint min_value(0);\n }\n\n.. eql:constraint:: std::min_ex_value(min: anytype)\n\n Specifies a non-inclusive lower bound for the value.\n\n Example:\n\n .. code-block:: sdl\n\n scalar type PositiveFloat extending float64 {\n constraint min_ex_value(0);\n }\n\n In this example, in contrast to the ``min_value`` constraint, a value\n of the ``PositiveFloat`` type cannot be ``0`` since the valid range of\n ``mix_ex_value`` does not include the value specified in the constraint.\n\n.. eql:constraint:: std::min_len_value(min: int64)\n\n Specifies the minimum allowed length of a value.\n\n Example:\n\n .. code-block:: sdl\n\n scalar type EmailAddress extending str {\n constraint min_len_value(3);\n }\n\n.. eql:constraint:: std::regexp(pattern: str)\n\n Limits to string values matching a regular expression.\n\n Example:\n\n .. code-block:: sdl\n\n scalar type LettersOnly extending str {\n constraint regexp(r'[A-Za-z]*');\n }\n\n See our documentation on :ref:`regular expression patterns\n ` for more information on those.\n\n.. eql:constraint:: std::exclusive\n\n Specifies that the link or property value must be exclusive (unique).\n\n When applied to a ``multi`` link or property, the exclusivity constraint\n guarantees that for every object, the set of values held by a link or\n property does not intersect with any other such set in any other object\n of this type.\n\n This constraint is only valid for concrete links and properties.\n Scalar type definitions cannot include this constraint.\n\n This constraint has an additional effect of creating an\n implicit :ref:`index ` on a property.\n This means that there's no need to add explicit indexes\n for properties with this constraint.\n\n Example:\n\n .. code-block:: sdl\n\n type User {\n # Make sure user names are unique.\n required name: str {\n constraint exclusive;\n }\n # Already indexed, don't need to do this:\n # index on (.name)\n\n # Make sure none of the \"owned\" items belong\n # to any other user.\n multi owns: Item {\n constraint exclusive;\n }\n }\n\n Sometimes it may be necessary to create a type where each *combination*\n of properties is unique. This can be achieved by defining an\n ``exclusive`` constraint for the combination, rather than on each\n property:\n\n .. code-block:: sdl\n\n type UniqueCoordinates {\n required x: int64;\n required y: int64;\n\n # Each combination of x and y must be unique.\n constraint exclusive on ( (.x, .y) );\n }\n\n Any possible expression can appear in the ``on ()`` clause of\n the ``exclusive`` constraint as long as it adheres to the following:\n\n * The expression can only contain references to the immediate\n properties or links of the type.\n * No :ref:`backlinks ` or long paths are\n allowed.\n * Only ``Immutable`` functions are allowed in the constraint\n expression.\n\n.. list-table::\n :class: seealso\n\n * - **See also**\n * - :ref:`Schema > Constraints `\n * - :ref:`SDL > Constraints `\n * - :ref:`DDL > Constraints `\n * - :ref:`Introspection > Constraints\n `\n" }, { "path": "docs/reference/stdlib/datetime.rst", "content": ".. _ref_std_datetime:\n\n\n===============\nDates and Times\n===============\n\n:edb-alt-title: Types, Functions, and Operators for Dates and Times\n\n.. list-table::\n :class: funcoptable\n\n * - :eql:type:`datetime`\n - Timezone-aware point in time\n\n * - :eql:type:`duration`\n - Absolute time span\n\n * - :eql:type:`cal::local_datetime`\n - Date and time w/o timezone\n\n * - :eql:type:`cal::local_date`\n - Date type\n\n * - :eql:type:`cal::local_time`\n - Time type\n\n * - :eql:type:`cal::relative_duration`\n - Relative time span\n\n * - :eql:type:`cal::date_duration`\n - Relative time span in days\n\n * - :eql:op:`dt + dt `\n - :eql:op-desc:`dtplus`\n\n * - :eql:op:`dt - dt `\n - :eql:op-desc:`dtminus`\n\n * - :eql:op:`= ` :eql:op:`\\!= ` :eql:op:`?= `\n :eql:op:`?!= ` :eql:op:`\\< ` :eql:op:`\\> `\n :eql:op:`\\<= ` :eql:op:`\\>= `\n - Comparison operators\n\n * - :eql:func:`to_str`\n - Render a date/time value to a string.\n\n * - :eql:func:`to_datetime`\n - :eql:func-desc:`to_datetime`\n\n * - :eql:func:`cal::to_local_datetime`\n - :eql:func-desc:`cal::to_local_datetime`\n\n * - :eql:func:`cal::to_local_date`\n - :eql:func-desc:`cal::to_local_date`\n\n * - :eql:func:`cal::to_local_time`\n - :eql:func-desc:`cal::to_local_time`\n\n * - :eql:func:`to_duration`\n - :eql:func-desc:`to_duration`\n\n * - :eql:func:`cal::to_relative_duration`\n - :eql:func-desc:`cal::to_relative_duration`\n\n * - :eql:func:`cal::to_date_duration`\n - :eql:func-desc:`cal::to_date_duration`\n\n * - :eql:func:`datetime_get`\n - :eql:func-desc:`datetime_get`\n\n * - :eql:func:`cal::time_get`\n - :eql:func-desc:`cal::time_get`\n\n * - :eql:func:`cal::date_get`\n - :eql:func-desc:`cal::date_get`\n\n * - :eql:func:`duration_get`\n - :eql:func-desc:`duration_get`\n\n * - :eql:func:`datetime_truncate`\n - :eql:func-desc:`datetime_truncate`\n\n * - :eql:func:`duration_truncate`\n - :eql:func-desc:`duration_truncate`\n\n * - :eql:func:`datetime_current`\n - :eql:func-desc:`datetime_current`\n\n * - :eql:func:`datetime_of_transaction`\n - :eql:func-desc:`datetime_of_transaction`\n\n * - :eql:func:`datetime_of_statement`\n - :eql:func-desc:`datetime_of_statement`\n\n * - :eql:func:`cal::duration_normalize_hours`\n - :eql:func-desc:`cal::duration_normalize_hours`\n\n * - :eql:func:`cal::duration_normalize_days`\n - :eql:func-desc:`cal::duration_normalize_days`\n\n.. _ref_std_datetime_intro:\n\n|Gel| offers two ways of representing date/time values:\n\n* a timezone-aware :eql:type:`std::datetime` type;\n\n* a set of \"local\" date/time types, not attached to any particular\n timezone: :eql:type:`cal::local_datetime`, :eql:type:`cal::local_date`,\n and :eql:type:`cal::local_time`.\n\nThere are also two different ways of measuring duration:\n\n* :eql:type:`duration` for using absolute and unambiguous units;\n\n* :eql:type:`cal::relative_duration` for using fuzzy units like years,\n months and days in addition to the absolute units.\n\nAll related operators, functions, and type casts are designed to maintain a\nstrict separation between timezone-aware and \"local\" date/time values.\n\n|Gel| stores and outputs timezone-aware values in UTC format.\n\n.. note::\n\n All date/time types are restricted to years between 1 and 9999, including\n the years 1 and 9999.\n\n Although many systems support ISO 8601 date/time formatting in theory,\n in practice the formatting before year 1 and after 9999 tends to\n be inconsistent. As such, dates outside this range are not reliably\n portable.\n\n.. _ref_std_datetime_timezones:\n\nTimezones\n---------\n\nFor timezone string literals, you may specify timezones in one of two ways:\n\n* IANA (Olson) timezone database name (e.g. ``America/New_York``)\n\n* A time zone abbreviation (e.g. ``EDT`` for Eastern Daylight Time)\n\nSee the `relevant section from the PostgreSQL documentation\n`_\nfor more detail about how time zones affect the behavior of date/time\nfunctionality.\n\n.. note::\n\n The IANA timezone database is maintained by Paul Eggert for the IANA. You can\n find a `GitHub repository with the latest timezone data here\n `_, and the `list of timezone names here\n `_.\n\n\n----------\n\n\n.. eql:type:: std::datetime\n\n\n Represents a timezone-aware moment in time.\n\n All dates must correspond to dates that exist in the proleptic Gregorian\n calendar.\n\n :eql:op:`Casting ` is a simple way to obtain a\n :eql:type:`datetime` value in an expression:\n\n .. code-block:: edgeql\n\n select '2018-05-07T15:01:22.306916+00';\n select '2018-05-07T15:01:22+00';\n\n When casting ``datetime`` from strings, the string must follow\n the ISO 8601 format with a timezone included.\n\n .. code-block:: edgeql-repl\n\n db> select 'January 01 2019 UTC';\n InvalidValueError: invalid input syntax for type\n std::datetime: 'January 01 2019 UTC'\n Hint: Please use ISO8601 format. Alternatively \"to_datetime\"\n function provides custom formatting options.\n\n db> select '2019-01-01T15:01:22';\n InvalidValueError: invalid input syntax for type\n std::datetime: '2019-01-01T15:01:22'\n Hint: Please use ISO8601 format. Alternatively \"to_datetime\"\n function provides custom formatting options.\n\n All ``datetime`` values are restricted to the range from year 1 to 9999.\n\n For more information regarding interacting with this type, see\n :eql:func:`datetime_get`, :eql:func:`to_datetime`, and :eql:func:`to_str`.\n\n\n----------\n\n\n.. eql:type:: cal::local_datetime\n\n A type for representing a date and time without a timezone.\n\n :eql:op:`Casting ` is a simple way to obtain a\n :eql:type:`cal::local_datetime` value in an expression:\n\n .. code-block:: edgeql\n\n select '2018-05-07T15:01:22.306916';\n select '2018-05-07T15:01:22';\n\n When casting ``cal::local_datetime`` from strings, the string must follow\n the ISO 8601 format without timezone:\n\n .. code-block:: edgeql-repl\n\n db> select '2019-01-01T15:01:22+00';\n InvalidValueError: invalid input syntax for type\n cal::local_datetime: '2019-01-01T15:01:22+00'\n Hint: Please use ISO8601 format. Alternatively\n \"cal::to_local_datetime\" function provides custom formatting\n options.\n\n db> select 'January 01 2019';\n InvalidValueError: invalid input syntax for type\n cal::local_datetime: 'January 01 2019'\n Hint: Please use ISO8601 format. Alternatively\n \"cal::to_local_datetime\" function provides custom formatting\n options.\n\n All ``datetime`` values are restricted to the range from year 1 to 9999.\n\n For more information regarding interacting with this type, see\n :eql:func:`datetime_get`, :eql:func:`cal::to_local_datetime`, and\n :eql:func:`to_str`.\n\n\n----------\n\n\n.. eql:type:: cal::local_date\n\n A type for representing a date without a timezone.\n\n :eql:op:`Casting ` is a simple way to obtain a\n :eql:type:`cal::local_date` value in an expression:\n\n .. code-block:: edgeql\n\n select '2018-05-07';\n\n When casting ``cal::local_date`` from strings, the string must follow the\n ISO 8601 date format.\n\n For more information regarding interacting with this type, see\n :eql:func:`cal::date_get`, :eql:func:`cal::to_local_date`, and\n :eql:func:`to_str`.\n\n\n----------\n\n\n.. eql:type:: cal::local_time\n\n A type for representing a time without a timezone.\n\n :eql:op:`Casting ` is a simple way to obtain a\n :eql:type:`cal::local_time` value in an expression:\n\n .. code-block:: edgeql\n\n select '15:01:22.306916';\n select '15:01:22';\n\n When casting ``cal::local_time`` from strings, the string must follow the\n ISO 8601 time format.\n\n For more information regarding interacting with this type, see\n :eql:func:`cal::time_get`, :eql:func:`cal::to_local_time`, and\n :eql:func:`to_str`.\n\n\n----------\n\n\n\n.. _ref_datetime_duration:\n\n.. eql:type:: std::duration\n\n A type for representing a span of time.\n\n A :eql:type:`duration` is a fixed number of seconds and microseconds and\n isn't adjusted by timezone, length of month, or anything else in datetime\n calculations.\n\n When converting from a string, only units of ``'microseconds'``,\n ``'milliseconds'``, ``'seconds'``, ``'minutes'``, and ``'hours'`` are\n valid:\n\n .. code-block:: edgeql-repl\n\n db> select '45.6 seconds';\n {'0:00:45.6'}\n db> select '15 milliseconds';\n {'0:00:00.015'}\n db> select '48 hours 45 minutes';\n {'48:45:00'}\n db> select '11 months';\n gel error: InvalidValueError: invalid input syntax for type\n std::duration: '11 months'\n Hint: Units bigger than hours cannot be used for std::duration.\n\n All date/time types support the ``+`` and ``-`` arithmetic operations\n with durations:\n\n .. code-block:: edgeql-repl\n\n db> select '2019-01-01T00:00:00Z' - '24 hours';\n {'2018-12-31T00:00:00+00:00'}\n db> select '22:00' + '1 hour';\n {'23:00:00'}\n\n For more information regarding interacting with this type, see\n :eql:func:`to_duration`, and :eql:func:`to_str` and date/time\n :eql:op:`operators `.\n\n\n----------\n\n\n.. eql:type:: cal::relative_duration\n\n A type for representing a relative span of time.\n\n Unlike :eql:type:`std::duration`, ``cal::relative_duration`` is an\n imprecise form of measurement. When months and days are used, the same\n relative duration could have a different absolute duration depending on\n the date you're measuring from.\n\n For example 2020 was a leap year and had 366 days. Notice how the number\n of hours in each year below is different:\n\n .. code-block:: edgeql-repl\n\n db> with\n ... first_day_of_2020 := '2020-01-01T00:00:00Z',\n ... one_year := '1 year',\n ... first_day_of_next_year := first_day_of_2020 + one_year\n ... select first_day_of_next_year - first_day_of_2020;\n {'8784:00:00'}\n db> with\n ... first_day_of_2019 := '2019-01-01T00:00:00Z',\n ... one_year := '1 year',\n ... first_day_of_next_year := first_day_of_2019 + one_year\n ... select first_day_of_next_year - first_day_of_2019;\n {'8760:00:00'}\n\n When converting from a string, only the following units are valid:\n\n - ``'microseconds'``\n - ``'milliseconds'``\n - ``'seconds'``\n - ``'minutes'``\n - ``'hours'``\n - ``'days'``\n - ``'weeks'``\n - ``'months'``\n - ``'years'``\n - ``'decades'``\n - ``'centuries'``\n - ``'millennia'``\n\n Examples of units usage:\n\n .. code-block:: edgeql\n\n select '45.6 seconds';\n select '15 milliseconds';\n select '3 weeks 45 minutes';\n select '-7 millennia';\n\n All date/time types support the ``+`` and ``-`` arithmetic operations\n with ``relative_duration``:\n\n .. code-block:: edgeql-repl\n\n db> select '2019-01-01T00:00:00Z' -\n ... '3 years';\n {'2016-01-01T00:00:00+00:00'}\n db> select '22:00' +\n ... '1 hour';\n {'23:00:00'}\n\n If an arithmetic operation results in a day that doesn't exist in the\n given month, the last day of the month will be used instead:\n\n .. code-block:: edgeql-repl\n\n db> select \"2021-01-31T15:00:00\" +\n ... \"1 month\";\n {'2021-02-28T15:00:00'}\n\n For arithmetic operations involving a ``cal::relative_duration``\n consisting of multiple components (units), higher-order components are\n applied first followed by lower-order components.\n\n .. code-block:: edgeql-repl\n\n db> select \"2021-04-30T15:00:00\" +\n ... \"1 month 1 day\";\n {'2021-05-31T15:00:00'}\n\n If you add the same components split into separate durations, adding the\n higher-order units first followed by the lower-order units, the\n calculation produces the same result as in the previous example:\n\n .. code-block:: edgeql-repl\n\n db> select \"2021-04-30T15:00:00\" +\n ... \"1 month\" +\n ... \"1 day\";\n {'2021-05-31T15:00:00'}\n\n When the order of operations is reversed, the result may be different for\n some corner cases:\n\n .. code-block:: edgeql-repl\n\n db> select \"2021-04-30T15:00:00\" +\n ... \"1 day\" +\n ... \"1 month\";\n {'2021-06-01T15:00:00'}\n\n .. rubric:: Gotchas\n\n Due to the implementation of ``relative_duration`` logic, arithmetic\n operations may behave counterintuitively.\n\n **Non-associative**\n\n .. code-block:: edgeql-repl\n\n db> select '2021-01-31T00:00:00' +\n ... '1 month' +\n ... '1 month';\n {'2021-03-28T00:00:00'}\n db> select '2021-01-31T00:00:00' +\n ... ('1 month' +\n ... '1 month');\n {'2021-03-31T00:00:00'}\n\n **Lossy**\n\n .. code-block:: edgeql-repl\n\n db> with m := '1 month'\n ... select '2021-01-31' + m\n ... =\n ... '2021-01-30' + m;\n {true}\n\n **Asymmetric**\n\n .. code-block:: edgeql-repl\n\n db> with m := '1 month'\n ... select '2021-01-31' + m - m;\n {'2021-01-28'}\n\n **Non-monotonic**\n\n .. code-block:: edgeql-repl\n\n db> with m := '1 month'\n ... select '2021-01-31T01:00:00' + m\n ... <\n ... '2021-01-30T23:00:00' + m;\n {true}\n db> with m := '2 month'\n ... select '2021-01-31T01:00:00' + m\n ... <\n ... '2021-01-30T23:00:00' + m;\n {false}\n\n For more information regarding interacting with this type, see\n :eql:func:`cal::to_relative_duration`, and :eql:func:`to_str` and\n date/time :eql:op:`operators `.\n\n\n----------\n\n\n.. eql:type:: cal::date_duration\n\n A type for representing a span of time in days.\n\n This type is similar to :eql:type:`cal::relative_duration`, except it only\n uses 2 units: months and days. It is the result of subtracting one\n :eql:type:`cal::local_date` from another. The purpose of this type is to\n allow performing ``+`` and ``-`` operations on a\n :eql:type:`cal::local_date` and to produce a :eql:type:`cal::local_date`\n as the result:\n\n .. code-block:: edgeql-repl\n\n db> select '2022-06-30' -\n ... '2022-06-25';\n {'P5D'}\n db> select '2022-06-25' +\n ... '5 days';\n {'2022-06-30'}\n db> select '2022-06-25' -\n ... '5 days';\n {'2022-06-20'}\n\n\n When converting from a string, only the following units are valid:\n\n - ``'days'``,\n - ``'weeks'``,\n - ``'months'``,\n - ``'years'``,\n - ``'decades'``,\n - ``'centuries'``,\n - ``'millennia'``.\n\n .. code-block:: edgeql\n\n select '45 days';\n select '3 weeks 5 days';\n select '-7 millennia';\n\n In most cases, ``date_duration`` is fully compatible with\n :eql:type:`cal::relative_duration` and shares the same general behavior\n and caveats. Gel will apply type coercion in the event it expects a\n :eql:type:`cal::relative_duration` and finds a ``cal::date_duration``\n instead.\n\n For more information regarding interacting with this type, see\n :eql:func:`cal::to_date_duration` and date/time :eql:op:`operators\n `.\n\n\n----------\n\n\n.. eql:operator:: dtplus: datetime + duration -> datetime\n datetime + cal::relative_duration \\\n -> cal::relative_duration\n duration + duration -> duration\n duration + cal::relative_duration \\\n -> cal::relative_duration\n cal::relative_duration + cal::relative_duration \\\n -> cal::relative_duration\n cal::local_datetime + cal::relative_duration \\\n -> cal::relative_duration\n cal::local_datetime + duration \\\n -> cal::local_datetime\n cal::local_time + cal::relative_duration \\\n -> cal::relative_duration\n cal::local_time + duration -> cal::local_time\n cal::local_date + cal::date_duration \\\n -> cal::local_date\n cal::date_duration + cal::date_duration \\\n -> cal::date_duration\n cal::local_date + cal::relative_duration \\\n -> cal::local_datetime\n cal::local_date + duration -> cal::local_datetime\n\n .. api-index:: Ā§datetime | duration Ā§+Ā§ datetime | durationĀ§\n\n Adds a duration and any other datetime value.\n\n This operator is commutative.\n\n .. code-block:: edgeql-repl\n\n db> select '22:00' + '1 hour';\n {'23:00:00'}\n db> select '1 hour' + '22:00';\n {'23:00:00'}\n db> select '1 hour' + '2 hours';\n {10800s}\n\n\n----------\n\n\n.. eql:operator:: dtminus: duration - duration -> duration\n datetime - datetime -> duration\n datetime - duration -> datetime\n datetime - cal::relative_duration -> datetime\n cal::relative_duration - cal::relative_duration \\\n -> cal::relative_duration\n cal::local_datetime - cal::local_datetime \\\n -> cal::relative_duration\n cal::local_datetime - cal::relative_duration \\\n -> cal::local_datetime\n cal::local_datetime - duration \\\n -> cal::local_datetime\n cal::local_time - cal::local_time \\\n -> cal::relative_duration\n cal::local_time - cal::relative_duration \\\n -> cal::local_time\n cal::local_time - duration -> cal::local_time\n cal::date_duration - cal::date_duration \\\n -> cal::date_duration\n cal::local_date - cal::local_date \\\n -> cal::date_duration\n cal::local_date - cal::date_duration \\\n -> cal::local_date\n cal::local_date - cal::relative_duration \\\n -> cal::local_datetime\n cal::local_date - duration -> cal::local_datetime\n duration - cal::relative_duration \\\n -> cal::relative_duration\n cal::relative_duration - duration\\\n -> cal::relative_duration\n\n .. api-index:: Ā§datetime | duration Ā§-Ā§ datetime | durationĀ§\n\n Subtracts two compatible datetime or duration values.\n\n .. code-block:: edgeql-repl\n\n db> select '2019-01-01T01:02:03+00' -\n ... '24 hours';\n {'2018-12-31T01:02:03Z'}\n db> select '2019-01-01T01:02:03+00' -\n ... '2019-02-01T01:02:03+00';\n {-2678400s}\n db> select '1 hour' -\n ... '2 hours';\n {-3600s}\n\n When subtracting a :eql:type:`cal::local_date` type from another, the\n result is given as a whole number of days using the\n :eql:type:`cal::date_duration` type:\n\n .. code-block:: edgeql-repl\n\n db> select '2022-06-25' -\n ... '2019-02-01';\n {'P1240D'}\n\n .. note::\n\n Subtraction doesn't make sense for some type combinations. You\n couldn't subtract a point in time from a duration, so neither can\n Gel (although the inverse ā€” subtracting a duration from a point in\n time ā€” is perfectly fine). You also couldn't subtract a timezone-aware\n datetime from a local one or vice versa. If you attempt any of these,\n Gel will raise an exception as shown in these examples.\n\n When subtracting a date/time object from a time interval, an exception\n will be raised:\n\n .. code-block:: edgeql-repl\n\n db> select '1 day' -\n ... '2019-01-01T01:02:03+00';\n QueryError: operator '-' cannot be applied to operands ...\n\n An exception will also be raised when trying to subtract a timezone-aware\n :eql:type:`std::datetime` type from :eql:type:`cal::local_datetime` or\n vice versa:\n\n .. code-block:: edgeql-repl\n\n db> select '2019-01-01T01:02:03+00' -\n ... '2019-02-01T01:02:03';\n QueryError: operator '-' cannot be applied to operands...\n db> select '2019-02-01T01:02:03' -\n ... '2019-01-01T01:02:03+00';\n QueryError: operator '-' cannot be applied to operands...\n\n\n----------\n\n.. eql:function:: std::datetime_current() -> datetime\n\n .. index:: now\n\n Returns the server's current date and time.\n\n .. code-block:: edgeql-repl\n\n db> select datetime_current();\n {'2018-05-14T20:07:11.755827Z'}\n\n This function is volatile since it always returns the current time when it\n is called. As a result, it cannot be used in :ref:`computed properties\n defined in schema `. This does *not* apply to\n computed properties outside of schema.\n\n----------\n\n\n.. eql:function:: std::datetime_of_transaction() -> datetime\n\n .. index:: now\n\n Returns the date and time of the start of the current transaction.\n\n This function is non-volatile since it returns the current time when the\n transaction is started, not when the function is called. As a result, it\n can be used in :ref:`computed properties ` defined\n in schema.\n\n----------\n\n\n.. eql:function:: std::datetime_of_statement() -> datetime\n\n .. index:: now\n\n Returns the date and time of the start of the current statement.\n\n This function is non-volatile since it returns the current time when the\n statement is started, not when the function is called. As a result, it\n can be used in :ref:`computed properties ` defined\n in schema.\n\n----------\n\n\n.. eql:function:: std::datetime_get(dt: datetime, el: str) -> float64\n std::datetime_get(dt: cal::local_datetime, \\\n el: str) -> float64\n\n Returns the element of a date/time given a unit name.\n\n You may pass any of these unit names for *el*:\n\n - ``'epochseconds'`` - the number of seconds since 1970-01-01 00:00:00\n UTC (Unix epoch) for :eql:type:`datetime` or local time for\n :eql:type:`cal::local_datetime`. It can be negative.\n - ``'century'`` - the century according to the Gregorian calendar\n - ``'day'`` - the day of the month (1-31)\n - ``'decade'`` - the decade (year divided by 10 and rounded down)\n - ``'dow'`` - the day of the week from Sunday (0) to Saturday (6)\n - ``'doy'`` - the day of the year (1-366)\n - ``'hour'`` - the hour (0-23)\n - ``'isodow'`` - the ISO day of the week from Monday (1) to Sunday (7)\n - ``'isoyear'`` - the ISO 8601 week-numbering year that the date falls in.\n See the ``'week'`` element for more details.\n - ``'microseconds'`` - the seconds including fractional value expressed\n as microseconds\n - ``'millennium'`` - the millennium. The third millennium started\n on Jan 1, 2001.\n - ``'milliseconds'`` - the seconds including fractional value expressed\n as milliseconds\n - ``'minutes'`` - the minutes (0-59)\n - ``'month'`` - the month of the year (1-12)\n - ``'quarter'`` - the quarter of the year (1-4)\n - ``'seconds'`` - the seconds, including fractional value from 0 up to and\n not including 60\n - ``'week'`` - the number of the ISO 8601 week-numbering week of\n the year. ISO weeks are defined to start on Mondays and the\n first week of a year must contain Jan 4 of that year.\n - ``'year'`` - the year\n\n .. code-block:: edgeql-repl\n\n db> select datetime_get(\n ... '2018-05-07T15:01:22.306916+00',\n ... 'epochseconds');\n {1525705282.306916}\n\n db> select datetime_get(\n ... '2018-05-07T15:01:22.306916+00',\n ... 'year');\n {2018}\n\n db> select datetime_get(\n ... '2018-05-07T15:01:22.306916+00',\n ... 'quarter');\n {2}\n\n db> select datetime_get(\n ... '2018-05-07T15:01:22.306916+00',\n ... 'doy');\n {127}\n\n db> select datetime_get(\n ... '2018-05-07T15:01:22.306916+00',\n ... 'hour');\n {15}\n\n\n----------\n\n\n.. eql:function:: cal::time_get(dt: cal::local_time, el: str) -> float64\n\n Returns the element of a time value given a unit name.\n\n You may pass any of these unit names for *el*:\n\n - ``'midnightseconds'``\n - ``'hour'``\n - ``'microseconds'``\n - ``'milliseconds'``\n - ``'minutes'``\n - ``'seconds'``\n\n For full description of what these elements extract see\n :eql:func:`datetime_get`.\n\n .. code-block:: edgeql-repl\n\n db> select cal::time_get(\n ... '15:01:22.306916', 'minutes');\n {1}\n\n db> select cal::time_get(\n ... '15:01:22.306916', 'milliseconds');\n {22306.916}\n\n\n----------\n\n\n.. eql:function:: cal::date_get(dt: local_date, el: str) -> float64\n\n Returns the element of a date given a unit name.\n\n The :eql:type:`cal::local_date` scalar has the following elements\n available for extraction:\n\n - ``'century'`` - the century according to the Gregorian calendar\n - ``'day'`` - the day of the month (1-31)\n - ``'decade'`` - the decade (year divided by 10 and rounded down)\n - ``'dow'`` - the day of the week from Sunday (0) to Saturday (6)\n - ``'doy'`` - the day of the year (1-366)\n - ``'isodow'`` - the ISO day of the week from Monday (1) to Sunday (7)\n - ``'isoyear'`` - the ISO 8601 week-numbering year that the date falls in.\n See the ``'week'`` element for more details.\n - ``'millennium'`` - the millennium. The third millennium started\n on Jan 1, 2001.\n - ``'month'`` - the month of the year (1-12)\n - ``'quarter'`` - the quarter of the year (1-4)\n not including 60\n - ``'week'`` - the number of the ISO 8601 week-numbering week of\n the year. ISO weeks are defined to start on Mondays and the\n first week of a year must contain Jan 4 of that year.\n - ``'year'`` - the year\n\n .. code-block:: edgeql-repl\n\n db> select cal::date_get(\n ... '2018-05-07', 'century');\n {21}\n\n db> select cal::date_get(\n ... '2018-05-07', 'year');\n {2018}\n\n db> select cal::date_get(\n ... '2018-05-07', 'month');\n {5}\n\n db> select cal::date_get(\n ... '2018-05-07', 'doy');\n {127}\n\n\n----------\n\n\n.. eql:function:: std::duration_get(dt: duration, el: str) -> float64\n std::duration_get(dt: cal::relative_duration, \\\n el: str) -> float64\n std::duration_get(dt: cal::date_duration, \\\n el: str) -> float64\n\n Returns the element of a duration given a unit name.\n\n You may pass any of these unit names as ``el``:\n\n - ``'millennium'`` - number of 1000-year chunks rounded down\n - ``'century'`` - number of centuries rounded down\n - ``'decade'`` - number of decades rounded down\n - ``'year'`` - number of years rounded down\n - ``'quarter'``- remaining quarters after whole years are accounted for\n - ``'month'`` - number of months left over after whole years are\n accounted for\n - ``'day'`` - number of days recorded in the duration\n - ``'hour'`` - number of hours\n - ``'minutes'`` - remaining minutes after whole hours are accounted for\n - ``'seconds'`` - remaining seconds, including fractional value after whole\n minutes are accounted for\n - ``'milliseconds'`` - remaining seconds including fractional value\n expressed as milliseconds\n - ``'microseconds'`` - remaining seconds including fractional value\n expressed as microseconds\n\n .. note ::\n\n Only for units ``'month'`` or larger or for units ``'hour'`` or smaller\n will you receive a total across multiple units expressed in the original\n duration. See *Gotchas* below for details.\n\n Additionally, it's possible to convert a given duration into seconds:\n\n - ``'totalseconds'`` - the number of seconds represented by the duration.\n It will be approximate for :eql:type:`cal::relative_duration` and\n :eql:type:`cal::date_duration` for units ``'month'`` or larger because a\n month is assumed to be 30 days exactly.\n\n The :eql:type:`duration` scalar has only ``'hour'`` and smaller units\n available for extraction.\n\n The :eql:type:`cal::relative_duration` scalar has all of the units\n available for extraction.\n\n The :eql:type:`cal::date_duration` scalar only has ``'date'`` and larger\n units available for extraction.\n\n .. code-block:: edgeql-repl\n\n db> select duration_get(\n ... '400 months', 'year');\n {33}\n db> select duration_get(\n ... '400 months', 'month');\n {4}\n db> select duration_get(\n ... '1 month 20 days 30 hours',\n ... 'day');\n {20}\n db> select duration_get(\n ... '30 hours', 'hour');\n {30}\n db> select duration_get(\n ... '1 month 20 days 30 hours',\n ... 'hour');\n {30}\n db> select duration_get('30 hours', 'hour');\n {30}\n db> select duration_get(\n ... '1 month 20 days 30 hours',\n ... 'totalseconds');\n {4428000}\n db> select duration_get(\n ... '30 hours', 'totalseconds');\n {108000}\n\n .. rubric:: Gotchas\n\n This function will provide you with a calculated total for the unit passed\n as ``el``, but only within the given \"size class\" of the unit. These size\n classes exist because they are logical breakpoints that we can't reliably\n convert values across. A month might be 30 days long, or it might be 28 or\n 29 or 31. A day is generally 24 hours, but with daylight savings, it might\n be longer or shorter.\n\n As a result, it's impossible to convert across these lines in a way that\n works in every situation. For some use cases, assuming a 30 day month works\n fine. For others, it might not. The size classes are as follows:\n\n - ``'month'`` and larger\n - ``'day'``\n - ``'hour'`` and smaller\n\n For example, if you specify ``'day'`` as your ``el`` argument, the function\n will return only the number of days expressed as ``N days`` in your\n duration. It will not add another day to the returned count for every 24\n hours (defined as ``24 hours``) in the duration, nor will it consider the\n months' constituent day counts in the returned value. Specifying\n ``'decade'`` for ``el`` will total up all decades represented in units\n ``'month'`` and larger, but it will not add a decade's worth of days to the\n returned value as an additional decade.\n\n In this example, the duration represents more than a day's time, but since\n ``'day'`` and ``'hour'`` are in different size classes, the extra day\n stemming from the duration's hours is not added.\n\n .. code-block:: edgeql-repl\n\n db> select duration_get(\n ... '1 day 36 hours', 'day');\n {1}\n\n In this counter example, both the decades and months are pooled together\n since they are in the same size class. The return value is 5: the 2\n ``'decades'`` and the 3 decades in ``'400 months'``.\n\n .. code-block:: edgeql-repl\n\n db> select duration_get(\n ... '2 decades 400 months', 'decade');\n {5}\n\n If a unit from a smaller size class would contribute to your desired unit's\n total, it is not added.\n\n .. code-block:: edgeql-repl\n\n db> select duration_get(\n ... '1 year 400 days', 'year');\n {1}\n\n When you request a unit in the smallest size class, it will be pooled with\n other durations in the same size class.\n\n .. code-block:: edgeql-repl\n\n db> select duration_get(\n ... '20 hours 3600 seconds', 'hour');\n {21}\n\n Seconds and smaller units always return remaining time in that unit after\n accounting for the next larger unit.\n\n .. code-block:: edgeql-repl\n\n db> select duration_get(\n ... '20 hours 3600 seconds', 'seconds');\n {0}\n db> select duration_get(\n ... '20 hours 3630 seconds', 'seconds');\n {30}\n\n Normalization and truncation may help you deal with this. If your use case\n allows for making assumptions about the duration of a month or a day, you\n can make those conversions for yourself using the\n :eql:func:`cal::duration_normalize_hours` or\n :eql:func:`cal::duration_normalize_days` functions. If you got back a\n duration as a result of a datetime calculation and don't need the level of\n granularity you have, you can truncate the value with\n :eql:func:`duration_truncate`.\n\n----------\n\n\n.. eql:function:: std::datetime_truncate(dt: datetime, unit: str) -> datetime\n\n Truncates the input datetime to a particular precision.\n\n The valid units in order or decreasing precision are:\n\n - ``'microseconds'``\n - ``'milliseconds'``\n - ``'seconds'``\n - ``'minutes'``\n - ``'hours'``\n - ``'days'``\n - ``'weeks'``\n - ``'months'``\n - ``'quarters'``\n - ``'years'``\n - ``'decades'``\n - ``'centuries'``\n\n .. code-block:: edgeql-repl\n\n db> select datetime_truncate(\n ... '2018-05-07T15:01:22.306916+00', 'years');\n {'2018-01-01T00:00:00Z'}\n\n db> select datetime_truncate(\n ... '2018-05-07T15:01:22.306916+00', 'quarters');\n {'2018-04-01T00:00:00Z'}\n\n db> select datetime_truncate(\n ... '2018-05-07T15:01:22.306916+00', 'days');\n {'2018-05-07T00:00:00Z'}\n\n db> select datetime_truncate(\n ... '2018-05-07T15:01:22.306916+00', 'hours');\n {'2018-05-07T15:00:00Z'}\n\n\n----------\n\n\n.. eql:function:: std::duration_truncate(dt: duration, unit: str) -> duration\n std::duration_truncate(dt: cal::relative_duration, \\\n unit: str) -> cal::relative_duration\n\n Truncates the input duration to a particular precision.\n\n The valid units for :eql:type:`duration` are:\n\n - ``'microseconds'``\n - ``'milliseconds'``\n - ``'seconds'``\n - ``'minutes'``\n - ``'hours'``\n\n In addition to the above the following are also valid for\n :eql:type:`cal::relative_duration`:\n\n - ``'days'``\n - ``'weeks'``\n - ``'months'``\n - ``'years'``\n - ``'decades'``\n - ``'centuries'``\n\n .. code-block:: edgeql-repl\n\n db> select duration_truncate(\n ... '15:01:22', 'hours');\n {'15:00:00'}\n db> select duration_truncate(\n ... '15:01:22.306916', 'minutes');\n {'15:01:00'}\n db> select duration_truncate(\n ... '400 months', 'years');\n {'P33Y'}\n db> select duration_truncate(\n ... '400 months', 'decades');\n {'P30Y'}\n\n\n----------\n\n\n.. eql:function:: std::to_datetime(s: str, fmt: optional str={}) -> datetime\n std::to_datetime(local: cal::local_datetime, zone: str) \\\n -> datetime\n std::to_datetime(year: int64, month: int64, day: int64, \\\n hour: int64, min: int64, sec: float64, zone: str) \\\n -> datetime\n std::to_datetime(epochseconds: decimal) -> datetime\n std::to_datetime(epochseconds: float64) -> datetime\n std::to_datetime(epochseconds: int64) -> datetime\n\n .. index:: parse datetime\n\n Create a :eql:type:`datetime` value.\n\n The :eql:type:`datetime` value can be parsed from the input\n :eql:type:`str` *s*. By default, the input is expected to conform\n to ISO 8601 format. However, the optional argument *fmt* can\n be used to override the :ref:`input format\n ` to other forms.\n\n .. code-block:: edgeql-repl\n\n db> select to_datetime('2018-05-07T15:01:22.306916+00');\n {'2018-05-07T15:01:22.306916Z'}\n db> select to_datetime('2018-05-07T15:01:22+00');\n {'2018-05-07T15:01:22Z'}\n db> select to_datetime('May 7th, 2018 15:01:22 +00',\n ... 'Mon DDth, YYYY HH24:MI:SS TZH');\n {'2018-05-07T15:01:22Z'}\n\n Alternatively, the :eql:type:`datetime` value can be constructed\n from a :eql:type:`cal::local_datetime` value:\n\n .. code-block:: edgeql-repl\n\n db> select to_datetime(\n ... '2019-01-01T01:02:03', 'HKT');\n {'2018-12-31T17:02:03Z'}\n\n Another way to construct a the :eql:type:`datetime` value is to specify it\n in terms of its component parts: year, month, day, hour, min, sec, and\n :ref:`zone `.\n\n .. code-block:: edgeql-repl\n\n db> select to_datetime(\n ... 2018, 5, 7, 15, 1, 22.306916, 'UTC');\n {'2018-05-07T15:01:22.306916000Z'}\n\n Finally, it is also possible to convert a Unix timestamp to a\n :eql:type:`datetime`\n\n .. code-block:: edgeql-repl\n\n db> select to_datetime(1590595184.584);\n {'2020-05-27T15:59:44.584000000Z'}\n\n------------\n\n\n.. eql:function:: cal::to_local_datetime(s: str, fmt: optional str={}) \\\n -> local_datetime\n cal::to_local_datetime(dt: datetime, zone: str) \\\n -> local_datetime\n cal::to_local_datetime(year: int64, month: int64, \\\n day: int64, hour: int64, min: int64, sec: float64) \\\n -> local_datetime\n\n .. index:: parse local_datetime\n\n Create a :eql:type:`cal::local_datetime` value.\n\n Similar to :eql:func:`to_datetime`, the :eql:type:`cal::local_datetime`\n value can be parsed from the input :eql:type:`str` *s* with an\n optional *fmt* argument or it can be given in terms of its\n component parts: *year*, *month*, *day*, *hour*, *min*, *sec*.\n\n For more details on formatting see :ref:`here\n `.\n\n .. code-block:: edgeql-repl\n\n db> select cal::to_local_datetime('2018-05-07T15:01:22.306916');\n {'2018-05-07T15:01:22.306916'}\n db> select cal::to_local_datetime('May 7th, 2018 15:01:22',\n ... 'Mon DDth, YYYY HH24:MI:SS');\n {'2018-05-07T15:01:22'}\n db> select cal::to_local_datetime(\n ... 2018, 5, 7, 15, 1, 22.306916);\n {'2018-05-07T15:01:22.306916'}\n\n A timezone-aware :eql:type:`datetime` type can be converted to local\n datetime in the specified :ref:`timezone `:\n\n .. code-block:: edgeql-repl\n\n db> select cal::to_local_datetime(\n ... '2018-12-31T22:00:00+08',\n ... 'America/Chicago');\n {'2018-12-31T08:00:00'}\n db> select cal::to_local_datetime(\n ... '2018-12-31T22:00:00+08',\n ... 'CST');\n {'2018-12-31T08:00:00'}\n\n\n------------\n\n\n.. eql:function:: cal::to_local_date(s: str, fmt: optional str={}) \\\n -> cal::local_date\n cal::to_local_date(dt: datetime, zone: str) \\\n -> cal::local_date\n cal::to_local_date(year: int64, month: int64, \\\n day: int64) -> cal::local_date\n\n .. index:: parse local_date\n\n Create a :eql:type:`cal::local_date` value.\n\n Similar to :eql:func:`to_datetime`, the :eql:type:`cal::local_date`\n value can be parsed from the input :eql:type:`str` *s* with an\n optional *fmt* argument or it can be given in terms of its\n component parts: *year*, *month*, *day*.\n\n For more details on formatting see :ref:`here\n `.\n\n .. code-block:: edgeql-repl\n\n db> select cal::to_local_date('2018-05-07');\n {'2018-05-07'}\n db> select cal::to_local_date('May 7th, 2018', 'Mon DDth, YYYY');\n {'2018-05-07'}\n db> select cal::to_local_date(2018, 5, 7);\n {'2018-05-07'}\n\n A timezone-aware :eql:type:`datetime` type can be converted to local date\n in the specified :ref:`timezone `:\n\n .. code-block:: edgeql-repl\n\n db> select cal::to_local_date(\n ... '2018-12-31T22:00:00+08',\n ... 'America/Chicago');\n {'2019-01-01'}\n\n\n------------\n\n\n.. eql:function:: cal::to_local_time(s: str, fmt: optional str={}) \\\n -> local_time\n cal::to_local_time(dt: datetime, zone: str) \\\n -> local_time\n cal::to_local_time(hour: int64, min: int64, sec: float64) \\\n -> local_time\n\n .. index:: parse local_time\n\n Create a :eql:type:`cal::local_time` value.\n\n Similar to :eql:func:`to_datetime`, the :eql:type:`cal::local_time`\n value can be parsed from the input :eql:type:`str` *s* with an\n optional *fmt* argument or it can be given in terms of its\n component parts: *hour*, *min*, *sec*.\n\n For more details on formatting see :ref:`here\n `.\n\n .. code-block:: edgeql-repl\n\n db> select cal::to_local_time('15:01:22.306916');\n {'15:01:22.306916'}\n db> select cal::to_local_time('03:01:22pm', 'HH:MI:SSam');\n {'15:01:22'}\n db> select cal::to_local_time(15, 1, 22.306916);\n {'15:01:22.306916'}\n\n A timezone-aware :eql:type:`datetime` type can be converted to local date\n in the specified :ref:`timezone `:\n\n .. code-block:: edgeql-repl\n\n db> select cal::to_local_time(\n ... '2018-12-31T22:00:00+08',\n ... 'America/Los_Angeles');\n {'06:00:00'}\n\n\n------------\n\n\n.. eql:function:: std::to_duration( \\\n named only hours: int64=0, \\\n named only minutes: int64=0, \\\n named only seconds: float64=0, \\\n named only microseconds: int64=0 \\\n ) -> duration\n\n .. index:: parse duration\n\n Create a :eql:type:`duration` value.\n\n This function uses ``named only`` arguments to create a\n :eql:type:`duration` value. The available duration fields are:\n *hours*, *minutes*, *seconds*, *microseconds*.\n\n .. code-block:: edgeql-repl\n\n db> select to_duration(hours := 1,\n ... minutes := 20,\n ... seconds := 45);\n {4845s}\n db> select to_duration(seconds := 4845);\n {4845s}\n\n\n.. eql:function:: std::duration_to_seconds(cur: duration) -> decimal\n\n Return duration as total number of seconds in interval.\n\n .. code-block:: edgeql-repl\n\n db> select duration_to_seconds('1 hour');\n {3600.000000n}\n db> select duration_to_seconds('10 second 123 ms');\n {10.123000n}\n\n\n------------\n\n\n.. eql:function:: cal::to_relative_duration( \\\n named only years: int64=0, \\\n named only months: int64=0, \\\n named only days: int64=0, \\\n named only hours: int64=0, \\\n named only minutes: int64=0, \\\n named only seconds: float64=0, \\\n named only microseconds: int64=0 \\\n ) -> cal::relative_duration\n\n .. index:: parse relative_duration\n\n Create a :eql:type:`cal::relative_duration` value.\n\n This function uses ``named only`` arguments to create a\n :eql:type:`cal::relative_duration` value. The available duration fields\n are: *years*, *months*, *days*, *hours*, *minutes*, *seconds*,\n *microseconds*.\n\n .. code-block:: edgeql-repl\n\n db> select cal::to_relative_duration(years := 5, minutes := 1);\n {'P5YT1S'}\n db> select cal::to_relative_duration(months := 3, days := 27);\n {'P3M27D'}\n\n\n------------\n\n\n.. eql:function:: cal::to_date_duration( \\\n named only years: int64=0, \\\n named only months: int64=0, \\\n named only days: int64=0 \\\n ) -> cal::date_duration\n\n .. index:: parse date_duration\n\n Create a :eql:type:`cal::date_duration` value.\n\n This function uses ``named only`` arguments to create a\n :eql:type:`cal::date_duration` value. The available duration fields\n are: *years*, *months*, *days*.\n\n .. code-block:: edgeql-repl\n\n db> select cal::to_date_duration(years := 1, days := 3);\n {'P1Y3D'}\n db> select cal::to_date_duration(days := 12);\n {'P12D'}\n\n\n------------\n\n\n.. eql:function:: cal::duration_normalize_hours( \\\n dur: cal::relative_duration \\\n ) -> cal::relative_duration\n\n .. index:: justify_hours\n\n Convert 24-hour chunks into days.\n\n This function converts all 24-hour chunks into day units. The resulting\n :eql:type:`cal::relative_duration` is guaranteed to have less than 24\n hours in total in the units smaler than days.\n\n .. code-block:: edgeql-repl\n\n db> select cal::duration_normalize_hours(\n ... '1312 hours');\n {'P54DT16H'}\n\n This is a lossless operation because 24 hours are always equal to 1 day\n in :eql:type:`cal::relative_duration` units.\n\n This is sometimes used together with\n :eql:func:`cal::duration_normalize_days`.\n\n------------\n\n\n.. eql:function:: cal::duration_normalize_days( \\\n dur: cal::relative_duration \\\n ) -> cal::relative_duration\n cal::duration_normalize_days( \\\n dur: cal::date_duration \\\n ) -> cal::date_duration\n\n .. index:: justify_days\n\n Convert 30-day chunks into months.\n\n This function converts all 30-day chunks into month units. The resulting\n :eql:type:`cal::relative_duration` or :eql:type:`cal::date_duration` is\n guaranteed to have less than 30 day units.\n\n .. code-block:: edgeql-repl\n\n db> select cal::duration_normalize_days(\n ... '1312 days');\n {'P3Y7M22D'}\n\n db> select cal::duration_normalize_days(\n ... '1312 days');\n {'P3Y7M22D'}\n\n This function is a form of approximation and does not preserve the exact\n duration.\n\n This is often used together with\n :eql:func:`cal::duration_normalize_hours`.\n" }, { "path": "docs/reference/stdlib/deprecated.rst", "content": ".. _ref_std_deprecated:\n\n==========\nDeprecated\n==========\n\n:edb-alt-title: Deprecated Functions\n\n\n.. list-table::\n :class: funcoptable\n\n * - :eql:func:`str_lpad`\n - :eql:func-desc:`str_lpad`\n\n * - :eql:func:`str_rpad`\n - :eql:func-desc:`str_rpad`\n\n * - :eql:func:`str_ltrim`\n - :eql:func-desc:`str_ltrim`\n\n * - :eql:func:`str_rtrim`\n - :eql:func-desc:`str_rtrim`\n\n\n----------\n\n\n.. eql:function:: std::str_lpad(string: str, n: int64, fill: str = ' ') -> str\n\n Return the input *string* left-padded to the length *n*.\n\n .. warning::\n\n This function is deprecated. Use\n :eql:func:`std::str_pad_start` instead.\n\n If the *string* is longer than *n*, then it is truncated to the\n first *n* characters. Otherwise, the *string* is padded on the\n left up to the total length *n* using *fill* characters (space by\n default).\n\n .. code-block:: edgeql-repl\n\n db> select str_lpad('short', 10);\n {' short'}\n db> select str_lpad('much too long', 10);\n {'much too l'}\n db> select str_lpad('short', 10, '.:');\n {'.:.:.short'}\n\n\n----------\n\n\n.. eql:function:: std::str_rpad(string: str, n: int64, fill: str = ' ') -> str\n\n Return the input *string* right-padded to the length *n*.\n\n .. warning::\n\n This function is deprecated. Use\n :eql:func:`std::str_pad_end` instead.\n\n If the *string* is longer than *n*, then it is truncated to the\n first *n* characters. Otherwise, the *string* is padded on the\n right up to the total length *n* using *fill* characters (space by\n default).\n\n .. code-block:: edgeql-repl\n\n db> select str_rpad('short', 10);\n {'short '}\n db> select str_rpad('much too long', 10);\n {'much too l'}\n db> select str_rpad('short', 10, '.:');\n {'short.:.:.'}\n\n\n----------\n\n\n.. eql:function:: std::str_ltrim(string: str, trim: str = ' ') -> str\n\n Return the input string with all leftmost *trim* characters removed.\n\n .. warning::\n\n This function is deprecated. Use\n :eql:func:`std::str_trim_start` instead.\n\n If the *trim* specifies more than one character they will be\n removed from the beginning of the *string* regardless of the order\n in which they appear.\n\n .. code-block:: edgeql-repl\n\n db> select str_ltrim(' data');\n {'data'}\n db> select str_ltrim('.....data', '.:');\n {'data'}\n db> select str_ltrim(':::::data', '.:');\n {'data'}\n db> select str_ltrim(':...:data', '.:');\n {'data'}\n db> select str_ltrim('.:.:.data', '.:');\n {'data'}\n\n\n----------\n\n\n.. eql:function:: std::str_rtrim(string: str, trim: str = ' ') -> str\n\n Return the input string with all rightmost *trim* characters removed.\n\n .. warning::\n\n This function is deprecated. Use\n :eql:func:`std::str_trim_end` instead.\n\n If the *trim* specifies more than one character they will be\n removed from the end of the *string* regardless of the order\n in which they appear.\n\n .. code-block:: edgeql-repl\n\n db> select str_rtrim('data ');\n {'data'}\n db> select str_rtrim('data.....', '.:');\n {'data'}\n db> select str_rtrim('data:::::', '.:');\n {'data'}\n db> select str_rtrim('data:...:', '.:');\n {'data'}\n db> select str_rtrim('data.:.:.', '.:');\n {'data'}\n\n----------\n\n.. eql:type:: cfg::DatabaseConfig\n\n The branch-level configuration object type.\n\n As of |EdgeDB| 5.0, this config object represents database *branch*\n and instance-level configuration.\n\n **Use the identical** :eql:type:`cfg::BranchConfig` instead.\n" }, { "path": "docs/reference/stdlib/enum.rst", "content": ".. _ref_std_enum:\n\n=====\nEnums\n=====\n\n:edb-alt-title: Enum Type\n\n\n.. eql:type:: std::enum\n\n An enumerated type is a data type consisting of an ordered list of values.\n\n An enum type can be declared in a schema by using the following\n syntax:\n\n .. code-block:: sdl\n\n scalar type Color extending enum;\n\n Enum values can then be accessed directly:\n\n .. code-block:: edgeql-repl\n\n db> select Color.Red is Color;\n {true}\n\n :eql:op:`Casting ` can be used to obtain an\n enum value in an expression:\n\n .. code-block:: edgeql-repl\n\n db> select 'Red' is Color;\n {false}\n db> select 'Red' is Color;\n {true}\n db> select 'Red' = Color.Red;\n {true}\n\n .. note::\n\n The enum values in EdgeQL are string-like in the fact that\n they can contain any characters that the strings can. This is\n different from some other languages where enum values are\n identifier-like and thus cannot contain some characters. For\n example, when working with GraphQL enum values that contain\n characters that aren't allowed in identifiers cannot be\n properly reflected. To address this, consider using only\n identifier-like enum values in cases where such compatibility\n is needed.\n\n Currently, enum values cannot be longer than 63 characters.\n" }, { "path": "docs/reference/stdlib/fts.rst", "content": ".. _ref_std_fts:\n\n.. versionadded:: 4.0\n\n================\nFull-text Search\n================\n\nThe ``fts`` built-in module contains various tools that enable full-text\nsearch functionality in Gel.\n\n.. note::\n\n Since full-text search is a natural language search, it may not be ideal\n for your use case, particularly if you want to find partial matches. In\n that case, you may want to look instead at :ref:`ref_ext_pgtrgm`.\n\n.. list-table::\n :class: funcoptable\n\n * - :eql:type:`fts::Language`\n - Common languages :eql:type:`enum`\n\n * - :eql:type:`fts::PGLanguage`\n - Postgres languages :eql:type:`enum`\n\n * - :eql:type:`fts::Weight`\n - Weight category :eql:type:`enum`\n\n * - :eql:type:`fts::document`\n - Opaque document type\n\n * - :eql:func:`fts::search`\n - :eql:func-desc:`fts::search`\n\n * - :eql:func:`fts::with_options`\n - :eql:func-desc:`fts::with_options`\n\nWhen considering FTS functionality our goal was to come up with an interface\nthat could support different backend FTS providers. To achieve that we've\nidentified the following components to the FTS functionality:\n\n1) Valid FTS targets must be indexed.\n2) The expected language should be specified at the time of creating an index.\n3) It should be possible to mark document parts as having different relevance.\n4) It should be possible to assign custom weights at runtime so as to make\n searching more flexible.\n5) The search query should be close to what people are already used to.\n\nTo address (1) we introduce a special ``fts::index``. The presence of this\nindex in a type declaration indicates that the type in question can be subject\nto full-text search. This is an unusual index as it actually affects the\nresults of :eql:func:`fts::search` function. This is unlike most indexes which\nonly affect the performance and not the actual results. Another special\nfeature of ``fts::index`` is that at most one such index can be declared per\ntype. If a type inherits this index from a parent and also declares its own,\nonly the new index applies and fully overrides the ``fts::index`` inherited\nfrom the parent type. This means that when dealing with a hierarchy of\nfull-text-searchable types, each type can customize what gets searched as\nneeded.\n\nThe language (2) is defined as part of the ``fts::index on`` expression. A\nspecial function :eql:func:`fts::with_options` is used for that purpose:\n\n.. code-block:: sdl\n\n type Item {\n required available: bool {\n default := false;\n };\n required name: str;\n description: str;\n\n index fts::index on (\n fts::with_options(\n .name,\n language := fts::Language.eng\n )\n );\n }\n\nThe above declaration specifies that ``Item`` is full-text-searchable,\nspecifically by examining the ``name`` property (and ignoring ``description``)\nand assuming that the contents of that property are in English.\n\nMarking different parts of the document as having different relevance (3) can\nalso be done by the :eql:func:`fts::with_options` function:\n\n.. code-block:: sdl\n\n type Item {\n required available: bool {\n default := false;\n };\n required name: str;\n description: str;\n\n index fts::index on ((\n fts::with_options(\n .name,\n language := fts::Language.eng,\n weight_category := fts::Weight.A,\n ),\n fts::with_options(\n .description,\n language := fts::Language.eng,\n weight_category := fts::Weight.B,\n )\n ));\n }\n\nThe schema now indicates that both ``name`` and ``description`` properties of\n``Item`` are full-text-searchable. Additionally, the ``name`` and\n``description`` have potentially different relevance.\n\nBy default :eql:func:`fts::search` assumes that the weight categories ``A``,\n``B``, ``C``, and ``D`` have the following weights: ``[1, 0.5, 0.25, 0.125]``.\nThis makes each successive category relevance score halved.\n\nConsider the following:\n\n.. code-block:: edgeql-repl\n\n gel> select Item{name, description};\n {\n default::Item {name: 'Canned corn', description: {}},\n default::Item {\n name: 'Candy corn',\n description: 'A great Halloween treat',\n },\n default::Item {\n name: 'Sweet',\n description: 'Treat made with corn sugar',\n },\n }\n\n gel> with res := (\n .... select fts::search(Item, 'corn treat', language := 'eng')\n .... )\n .... select res.object {name, description, score := res.score}\n .... order by res.score desc;\n {\n default::Item {\n name: 'Candy corn',\n description: 'A great Halloween treat',\n score: 0.4559453,\n },\n default::Item {\n name: 'Canned corn',\n description: {},\n score: 0.30396354,\n },\n default::Item {\n name: 'Sweet',\n description: 'Treat made with corn sugar',\n score: 0.30396354,\n },\n }\n\nAs you can see, the highest scoring match came from an ``Item`` that had the\nsearch terms appear in both ``name`` and ``description``. It is also apparent\nthat matching a single term from the search query in the ``name`` property\nscores the same as matching two terms in ``description`` as we would expect\nbased on their weight categories. We can, however, customize the weights (4)\nto change this trend:\n\n.. code-block:: edgeql-repl\n\n gel> with res := (\n .... select fts::search(\n .... Item, 'corn treat',\n .... language := 'eng',\n .... weights := [0.2, 1],\n .... )\n .... )\n .... select res.object {name, description, score := res.score}\n .... order by res.score desc;\n {\n default::Item {\n name: 'Sweet',\n description: 'Treat made with corn sugar',\n score: 0.6079271,\n },\n default::Item {\n name: 'Candy corn',\n description: 'A great Halloween treat',\n score: 0.36475626,\n },\n default::Item {\n name: 'Canned corn',\n description: {},\n score: 0.06079271,\n },\n }\n\nWe can even use custom weights to completely ignore one of the properties\n(e.g. ``name``) in our search, although we also need to add a filter based on\nthe score to make this work properly:\n\n.. code-block:: edgeql-repl\n\n gel> with res := (\n .... select fts::search(\n .... Item, 'corn treat',\n .... language := 'eng',\n .... weights := [0, 1],\n .... )\n .... )\n .... select res.object {name, description, score := res.score}\n .... filter res.score > 0\n .... order by res.score desc;\n {\n default::Item {\n name: 'Sweet',\n description: 'Treat made with corn sugar',\n score: 0.6079271,\n },\n default::Item {\n name: 'Candy corn',\n description: 'A great Halloween treat',\n score: 0.30396354,\n },\n }\n\nFinally, the search query supports features for fine-tuning (5). By default,\nall search terms are desirable, but ultimately optional. You can enclose a\nterm or even a phrase in ``\"...\"`` to indicate that this specific term is of\nincreased importance and should appear in all matches:\n\n.. code-block:: edgeql-repl\n\n gel> with res := (\n .... select fts::search(\n .... Item, '\"corn sugar\"',\n .... language := 'eng',\n .... )\n .... )\n .... select res.object {name, description, score := res.score}\n .... order by res.score desc;\n {\n default::Item {\n name: 'Sweet',\n description: 'Treat made with corn sugar',\n score: 0.4955161,\n },\n }\n\nOnly one ``Item`` contains the phrase \"corn sugar\" and incomplete matches are\nomitted.\n\nThe search query can also use ``AND`` (using upper-case to indicate that it is\na query modifier and not part of the query) to indicate whether terms are\nrequired or optional:\n\n.. code-block:: edgeql-repl\n\n gel> with res := (\n .... select fts::search(\n .... Item, 'sweet AND treat',\n .... language := 'eng',\n .... )\n .... )\n .... select res.object {name, description, score := res.score}\n .... order by res.score desc;\n {\n default::Item {\n name: 'Sweet',\n description: 'Treat made with corn sugar',\n score: 0.70076555,\n },\n }\n\nAdding a ``!`` in front of a search term marks it as something that\nthe matching object *must not* contain:\n\n.. code-block:: edgeql-repl\n\n gel> with res := (\n .... select fts::search(\n .... Item, '!treat',\n .... language := 'eng',\n .... )\n .... )\n .... select res.object {name, description, score := res.score}\n .... order by res.score desc;\n {\n default::Item {\n name: 'Canned corn',\n description: {},\n score: 0,\n },\n }\n\n----------\n\n\n.. eql:type:: fts::Language\n\n An :eql:type:`enum` for representing commonly supported languages.\n\n When indexing an object for full-text search it is important to specify\n the expected language by :eql:func:`fts::with_options` function. This\n particular :eql:type:`enum` represents languages that are common across\n several possible [future] backend implementations and thus are \"safe\" even\n if the backend implementation switches from one of the options to another.\n This generic enum is the recommended way of specifying the language.\n\n The following `ISO 639-3 `_ language identifiers are available:\n ``ara``, ``hye``, ``eus``, ``cat``, ``dan``, ``nld``, ``eng``, ``fin``,\n ``fra``, ``deu``, ``ell``, ``hin``, ``hun``, ``ind``, ``gle``, ``ita``,\n ``nor``, ``por``, ``ron``, ``rus``, ``spa``, ``swe``, ``tur``.\n\n----------\n\n\n.. eql:type:: fts::PGLanguage\n\n An :eql:type:`enum` for representing languages supported by PostgreSQL.\n\n When indexing an object for full-text search it is important to specify\n the expected language by :eql:func:`fts::with_options` function. This\n particular :eql:type:`enum` represents languages that are available in\n PostgreSQL implementation of full-text search.\n\n The following `ISO 639-3 `_ language identifiers are available:\n ``ara``, ``hye``, ``eus``, ``cat``, ``dan``, ``nld``, ``eng``, ``fin``,\n ``fra``, ``deu``, ``ell``, ``hin``, ``hun``, ``ind``, ``gle``, ``ita``,\n ``lit``, ``npi``, ``nor``, ``por``, ``ron``, ``rus``, ``srp``, ``spa``,\n ``swe``, ``tam``, ``tur``, ``yid``.\n\n Additionally, the ``xxx_simple`` identifier is also available to represent\n the ``pg_catalog.simple`` language setting.\n\n Unless you need some particular language setting that is not available in\n the :eql:type:`fts::Language`, it is recommended that you use the more\n general lanuguage enum instead.\n\n\n----------\n\n\n.. eql:type:: fts::Weight\n\n An :eql:type:`enum` for representing weight categories.\n\n When indexing an object for full-text search different properties of this\n object may have different significance. To account for that, they can be\n assigned different weight categories by using\n :eql:func:`fts::with_options` function. There are four available weight\n categories: ``A``, ``B``, ``C``, or ``D``.\n\n\n----------\n\n\n.. eql:type:: fts::document\n\n An opaque transient type used in ``fts::index``.\n\n This type is technically what the ``fts::index`` expects as a valid ``on``\n expression. It cannot be directly instantiated and can only be produced as\n the result of applying the special :eql:func:`fts::with_options` function.\n Thus this type only appears in full-text search index definitions and\n cannot appear as either a property type or anywhere in regular queries.\n\n\n------------\n\n\n.. eql:function:: fts::search( \\\n object: anyobject, \\\n query: str, \\\n named only language: str = fts::Language.eng, \\\n named only weights: optional array = {}, \\\n ) -> optional tuple\n\n Perform full-text search on a target object.\n\n This function applies the search ``query`` to the specified object. If a\n match is found, the result will consist of a tuple with the matched\n ``object`` and the corresponding ``score``. A higher ``score`` indicates a\n better match. In case no match is found, the function will return an empty\n set ``{}``. Likewise, ``{}`` is returned if the ``object`` has no\n ``fts::index`` defined for it.\n\n The ``language`` parameter can be specified in order to match the expected\n indexed language. In case of mismatch there is a big chance that the query\n will not produce the expected results.\n\n The optional ``weights`` parameter can be passed in order to provide\n custom weights to the different weight groups. By default, the weights are\n ``[1, 0.5, 0.25, 0.125]`` representing groups of diminishing significance.\n\n\n------------\n\n\n.. eql:function:: fts::with_options( \\\n text: str, \\\n NAMED ONLY language: anyenum, \\\n NAMED ONLY weight_category: optional fts::Weight = \\\n fts::Weight.A, \\\n ) -> fts::document\n\n Assign language and weight category to a document portion.\n\n This is a special function that can only appear inside ``fts::index``\n expressions.\n\n The ``text`` expression specifies the portion of the document that will be\n indexed and available for full-text search.\n\n The ``language`` parameter specifies the expected language of the ``text``\n expression. This affects how the index accounts for grammatical variants\n of a given word (e.g. how plural and singular forms are determined, etc.).\n\n The ``weight_category`` parameter assigns one of four available weight\n categories to the ``text`` expression: ``A``, ``B``, ``C``, or ``D``. By\n themselves, the categories simply group together portions of the document\n so that these groups can be ascribed different significance by the\n :eql:func:`fts::search` function. By default it is assumed that each\n successive category is half as significant as the previous, starting with\n ``A`` as the most significant. However, these default weights can be\n overridden when making a call to :eql:func:`fts::search`.\n\n\n.. _iso639: https://iso639-3.sil.org/code_tables/639/data\n" }, { "path": "docs/reference/stdlib/generic.rst", "content": ".. _ref_std_generic:\n\n=======\nGeneric\n=======\n\n:edb-alt-title: Generic Functions and Operators\n\n.. list-table::\n :class: funcoptable\n\n * - :eql:op:`anytype = anytype `\n - :eql:op-desc:`eq`\n\n * - :eql:op:`anytype != anytype `\n - :eql:op-desc:`neq`\n\n * - :eql:op:`anytype ?= anytype `\n - :eql:op-desc:`coaleq`\n\n * - :eql:op:`anytype ?!= anytype `\n - :eql:op-desc:`coalneq`\n\n * - :eql:op:`anytype \\< anytype `\n - :eql:op-desc:`lt`\n\n * - :eql:op:`anytype \\> anytype `\n - :eql:op-desc:`gt`\n\n * - :eql:op:`anytype \\<= anytype `\n - :eql:op-desc:`lteq`\n\n * - :eql:op:`anytype \\>= anytype `\n - :eql:op-desc:`gteq`\n\n * - :eql:func:`len`\n - :eql:func-desc:`len`\n\n * - :eql:func:`contains`\n - :eql:func-desc:`contains`\n\n * - :eql:func:`find`\n - :eql:func-desc:`find`\n\n.. note::\n\n In EdgeQL, any value can be compared to another as long as their types\n are compatible.\n\n\n-----------\n\n\n.. eql:operator:: eq: anytype = anytype -> bool\n\n .. index:: comparison\n .. api-index:: =\n\n Compares two values for equality.\n\n .. code-block:: edgeql-repl\n\n db> select 3 = 3.0;\n {true}\n db> select 3 = 3.14;\n {false}\n db> select [1, 2] = [1, 2];\n {true}\n db> select (1, 2) = (x := 1, y := 2);\n {true}\n db> select (x := 1, y := 2) = (a := 1, b := 2);\n {true}\n db> select 'hello' = 'world';\n {false}\n\n .. warning::\n\n When either operand in an equality comparison is an empty set, the\n result will not be a ``bool`` but instead an empty set.\n\n .. code-block:: edgeql-repl\n\n db> select true = {};\n {}\n\n If one of the operands in an equality comparison could be an empty set,\n you may want to use the :eql:op:`coalescing equality ` operator\n (``?=``) instead.\n\n----------\n\n\n.. eql:operator:: neq: anytype != anytype -> bool\n\n .. index:: not equal, comparison\n .. api-index:: !=\n\n Compares two values for inequality.\n\n .. code-block:: edgeql-repl\n\n\n db> select 3 != 3.0;\n {false}\n db> select 3 != 3.14;\n {true}\n db> select [1, 2] != [2, 1];\n {false}\n db> select (1, 2) != (x := 1, y := 2);\n {false}\n db> select (x := 1, y := 2) != (a := 1, b := 2);\n {false}\n db> select 'hello' != 'world';\n {true}\n\n .. warning::\n\n When either operand in an inequality comparison is an empty set, the\n result will not be a ``bool`` but instead an empty set.\n\n .. code-block:: edgeql-repl\n\n db> select true != {};\n {}\n\n If one of the operands in an inequality comparison could be an empty\n set, you may want to use the :eql:op:`coalescing inequality `\n operator (``?!=``) instead.\n\n\n----------\n\n\n.. eql:operator:: coaleq: optional anytype ?= optional anytype -> bool\n\n .. index:: coalesce equal, comparison, empty set\n .. api-index:: ?=\n\n Compares two (potentially empty) values for equality.\n\n This works the same as a regular :eql:op:`=` operator, but also allows\n comparing an empty ``{}`` set. Two empty sets are considered equal.\n\n .. code-block:: edgeql-repl\n\n db> select {1} ?= {1.0};\n {true}\n db> select {1} ?= {};\n {false}\n db> select {} ?= {};\n {true}\n\n\n----------\n\n\n.. eql:operator:: coalneq: optional anytype ?!= optional anytype -> bool\n\n .. index:: coalesce not equal, comparison\n .. api-index:: ?!=\n\n Compares two (potentially empty) values for inequality.\n\n This works the same as a regular :eql:op:`=` operator, but also allows\n comparing an empty ``{}`` set. Two empty sets are considered equal.\n\n .. code-block:: edgeql-repl\n\n db> select {2} ?!= {2};\n {false}\n db> select {1} ?!= {};\n {true}\n db> select {} ?!= {};\n {false}\n\n\n----------\n\n\n.. eql:operator:: lt: anytype < anytype -> bool\n\n .. index:: comparison\n .. api-index:: <\n\n Less than operator.\n\n The operator returns ``true`` if the value of the left expression is less\n than the value of the right expression:\n\n .. code-block:: edgeql-repl\n\n db> select 1 < 2;\n {true}\n db> select 2 < 2;\n {false}\n db> select 'hello' < 'world';\n {true}\n db> select (1, 'hello') < (1, 'world');\n {true}\n\n .. warning::\n\n When either operand in a comparison is an empty set, the result will\n not be a ``bool`` but instead an empty set.\n\n .. code-block:: edgeql-repl\n\n db> select 1 < {};\n {}\n\n If one of the operands in a comparison could be an empty set, you may\n want to coalesce the result of the comparison with ``false`` to ensure\n your result is boolean.\n\n .. code-block:: edgeql-repl\n\n db> select (1 < {}) ?? false;\n {false}\n\n\n----------\n\n\n.. eql:operator:: gt: anytype > anytype -> bool\n\n .. index:: comparison\n .. api-index:: >\n\n Greater than operator.\n\n The operator returns ``true`` if the value of the left expression is\n greater than the value of the right expression:\n\n .. code-block:: edgeql-repl\n\n db> select 1 > 2;\n {false}\n db> select 3 > 2;\n {true}\n db> select 'hello' > 'world';\n {false}\n db> select (1, 'hello') > (1, 'world');\n {false}\n\n .. warning::\n\n When either operand in a comparison is an empty set, the result will\n not be a ``bool`` but instead an empty set.\n\n .. code-block:: edgeql-repl\n\n db> select 1 > {};\n {}\n\n If one of the operands in a comparison could be an empty set, you may\n want to coalesce the result of the comparison with ``false`` to ensure\n your result is boolean.\n\n .. code-block:: edgeql-repl\n\n db> select (1 > {}) ?? false;\n {false}\n\n\n----------\n\n\n.. eql:operator:: lteq: anytype <= anytype -> bool\n\n .. index:: comparison\n .. api-index:: <=\n\n Less or equal operator.\n\n The operator returns ``true`` if the value of the left expression is less\n than or equal to the value of the right expression:\n\n .. code-block:: edgeql-repl\n\n db> select 1 <= 2;\n {true}\n db> select 2 <= 2;\n {true}\n db> select 3 <= 2;\n {false}\n db> select 'hello' <= 'world';\n {true}\n db> select (1, 'hello') <= (1, 'world');\n {true}\n\n .. warning::\n\n When either operand in a comparison is an empty set, the result will\n not be a ``bool`` but instead an empty set.\n\n .. code-block:: edgeql-repl\n\n db> select 1 <= {};\n {}\n\n If one of the operands in a comparison could be an empty set, you may\n want to coalesce the result of the comparison with ``false`` to ensure\n your result is boolean.\n\n .. code-block:: edgeql-repl\n\n db> select (1 <= {}) ?? false;\n {false}\n\n\n----------\n\n\n.. eql:operator:: gteq: anytype >= anytype -> bool\n\n .. index:: comparison\n .. api-index:: >=\n\n Greater or equal operator.\n\n The operator returns ``true`` if the value of the left expression is\n greater than or equal to the value of the right expression:\n\n .. code-block:: edgeql-repl\n\n db> select 1 >= 2;\n {false}\n db> select 2 >= 2;\n {true}\n db> select 3 >= 2;\n {true}\n db> select 'hello' >= 'world';\n {false}\n db> select (1, 'hello') >= (1, 'world');\n {false}\n\n .. warning::\n\n When either operand in a comparison is an empty set, the result will\n not be a ``bool`` but instead an empty set.\n\n .. code-block:: edgeql-repl\n\n db> select 1 >= {};\n {}\n\n If one of the operands in a comparison could be an empty set, you may\n want to coalesce the result of the comparison with ``false`` to ensure\n your result is boolean.\n\n .. code-block:: edgeql-repl\n\n db> select (1 >= {}) ?? false;\n {false}\n\n\n----------\n\n\n.. eql:function:: std::len(value: str) -> int64\n std::len(value: bytes) -> int64\n std::len(value: array) -> int64\n\n .. index:: length, count\n\n Returns the number of elements of a given value.\n\n This function works with the :eql:type:`str`, :eql:type:`bytes` and\n :eql:type:`array` types:\n\n .. code-block:: edgeql-repl\n\n db> select len('foo');\n {3}\n\n db> select len(b'bar');\n {3}\n\n db> select len([2, 5, 7]);\n {3}\n\n\n----------\n\n\n.. eql:function:: std::contains(haystack: str, needle: str) -> bool\n std::contains(haystack: bytes, needle: bytes) -> bool\n std::contains(haystack: array, needle: anytype) \\\n -> bool\n std::contains(haystack: range, \\\n needle: range) \\\n -> std::bool\n std::contains(haystack: range, \\\n needle: anypoint) \\\n -> std::bool\n std::contains(haystack: multirange, \\\n needle: multirange) \\\n -> std::bool\n std::contains(haystack: multirange, \\\n needle: range) \\\n -> std::bool\n std::contains(haystack: multirange, \\\n needle: anypoint) \\\n -> std::bool\n\n .. index:: find, strpos, includes\n\n Returns true if the given sub-value exists within the given value.\n\n When *haystack* is a :eql:type:`str` or a :eql:type:`bytes` value,\n this function will return ``true`` if it contains *needle* as a\n subsequence within it or ``false`` otherwise:\n\n .. code-block:: edgeql-repl\n\n db> select contains('qwerty', 'we');\n {true}\n\n db> select contains(b'qwerty', b'42');\n {false}\n\n When *haystack* is an :eql:type:`array`, the function will return\n ``true`` if the array contains the element specified as *needle* or\n ``false`` otherwise:\n\n .. code-block:: edgeql-repl\n\n db> select contains([2, 5, 7, 2, 100], 2);\n {true}\n\n When *haystack* is a :ref:`range `, the function will\n return ``true`` if it contains either the specified sub-range or element.\n The function will return ``false`` otherwise.\n\n .. code-block:: edgeql-repl\n\n db> select contains(range(1, 10), range(2, 5));\n {true}\n\n db> select contains(range(1, 10), range(2, 15));\n {false}\n\n db> select contains(range(1, 10), 2);\n {true}\n\n db> select contains(range(1, 10), 10);\n {false}\n\n When *haystack* is a :ref:`multirange `, the function\n will return ``true`` if it contains either the specified multirange,\n sub-range or element. The function will return ``false`` otherwise.\n\n .. code-block:: edgeql-repl\n\n db> select contains(\n ... multirange([\n ... range(1, 4), range(7),\n ... ]),\n ... multirange([\n ... range(1, 2), range(8, 10),\n ... ]),\n ... );\n {true}\n\n db> select contains(\n ... multirange([\n ... range(1, 4), range(8, 10),\n ... ]),\n ... range(8),\n ... );\n {false}\n\n db> select contains(\n ... multirange([\n ... range(1, 4), range(8, 10),\n ... ]),\n ... 3,\n ... );\n {true}\n\n When *haystack* is :ref:`JSON `, the function will return\n ``true`` if the json data contains the element specified as *needle* or\n ``false`` otherwise:\n\n .. code-block:: edgeql-repl\n\n db> with haystack := to_json('{\n ... \"city\": \"Baerlon\",\n ... \"city\": \"Caemlyn\"\n ... }'),\n ... needle := to_json('{\n ... \"city\": \"Caemlyn\"\n ... }'),\n ... select contains(haystack, needle);\n {true}\n\n\n----------\n\n\n.. eql:function:: std::find(haystack: str, needle: str) -> int64\n std::find(haystack: bytes, needle: bytes) -> int64\n std::find(haystack: array, needle: anytype, \\\n from_pos: int64=0) -> int64\n\n .. index:: find, strpos\n\n Returns the index of a given sub-value in a given value.\n\n When *haystack* is a :eql:type:`str` or a :eql:type:`bytes` value, the\n function will return the index of the first occurrence of *needle* in it.\n\n When *haystack* is an :eql:type:`array`, this will return the index of the\n the first occurrence of the element passed as *needle*. For\n :eql:type:`array` inputs it is also possible to provide an optional\n *from_pos* argument to specify the position from which to start the\n search.\n\n If the *needle* is not found, return ``-1``.\n\n .. code-block:: edgeql-repl\n\n db> select find('qwerty', 'we');\n {1}\n\n db> select find(b'qwerty', b'42');\n {-1}\n\n db> select find([2, 5, 7, 2, 100], 2);\n {0}\n\n db> select find([2, 5, 7, 2, 100], 2, 1);\n {3}\n" }, { "path": "docs/reference/stdlib/index.rst", "content": ".. versioned-section::\n\n.. _ref_std:\n\n================\nStandard Library\n================\n\n.. toctree::\n :maxdepth: 3\n :hidden:\n\n generic\n set\n type\n math\n string\n bool\n numbers\n json\n uuid\n enum\n datetime\n array\n tuple\n range\n bytes\n sequence\n objects\n abstract\n constraints\n net\n fts\n sys\n cfg\n pgcrypto\n pg_trgm\n pg_unaccent\n pgvector\n postgis\n deprecated\n\n|Gel| comes with a rigorously defined type system consisting of **scalar\ntypes**, **collection types** (like arrays and tuples), and **object types**.\nThere is also a library of built-in functions and operators for working with\neach datatype.\n\n\n.. _ref_datamodel_typesystem:\n\nScalar Types\n------------\n\n.. _ref_datamodel_scalar_types:\n\n*Scalar types* store primitive data.\n\n- :ref:`Strings `\n- :ref:`Numbers `\n- :ref:`Booleans `\n- :ref:`Dates and times `\n- :ref:`Enums `\n- :ref:`JSON `\n- :ref:`UUID `\n- :ref:`Bytes `\n- :ref:`Sequences `\n- :ref:`Abstract types `: these are the types that\n undergird the scalar hierarchy.\n\n.. _ref_datamodel_collection_types:\n\nCollection Types\n----------------\n\n*Collection types* are special generic types used to group homogeneous or\nheterogeneous data.\n\n- :ref:`Arrays `\n- :ref:`Tuples `\n\nRange Types\n-----------\n\n- :ref:`Range `\n- :ref:`Multirange `\n\nObject Types\n------------\n\n- :ref:`Object Types `\n\nTypes and Sets\n--------------\n\n- :ref:`Sets `\n- :ref:`Types `\n- :ref:`Casting `\n\nUtilities\n---------\n\n- :ref:`Math `\n- :ref:`Comparison `\n- :ref:`Constraints `\n- :ref:`Full-text Search `\n- :ref:`System `\n\nExtensions\n----------\n\n- :ref:`ext::pgvector `\n" }, { "path": "docs/reference/stdlib/json.rst", "content": ".. _ref_std_json:\n\n====\nJSON\n====\n\n:edb-alt-title: JSON Functions and Operators\n\n.. list-table::\n :class: funcoptable\n\n * - :eql:type:`json`\n - JSON scalar type\n\n * - :eql:op:`json[i] `\n - :eql:op-desc:`jsonidx`\n\n * - :eql:op:`json[from:to] `\n - :eql:op-desc:`jsonslice`\n\n * - :eql:op:`json ++ json `\n - :eql:op-desc:`jsonplus`\n\n * - :eql:op:`json[name] `\n - :eql:op-desc:`jsonobjdest`\n\n * - :eql:op:`= ` :eql:op:`\\!= ` :eql:op:`?= `\n :eql:op:`?!= ` :eql:op:`\\< ` :eql:op:`\\> `\n :eql:op:`\\<= ` :eql:op:`\\>= `\n - Comparison operators\n\n * - :eql:func:`to_json`\n - :eql:func-desc:`to_json`\n\n * - :eql:func:`to_str`\n - Render JSON value to a string.\n\n * - :eql:func:`json_get`\n - :eql:func-desc:`json_get`\n\n * - :eql:func:`json_set`\n - :eql:func-desc:`json_set`\n\n * - :eql:func:`json_array_unpack`\n - :eql:func-desc:`json_array_unpack`\n\n * - :eql:func:`json_object_pack`\n - :eql:func-desc:`json_object_pack`\n\n * - :eql:func:`json_object_unpack`\n - :eql:func-desc:`json_object_unpack`\n\n * - :eql:func:`json_typeof`\n - :eql:func-desc:`json_typeof`\n\n.. _ref_std_json_construction:\n\nConstructing JSON Values\n------------------------\n\nJSON in Gel is a :ref:`scalar type `. This type\ndoesn't have its own literal, and instead can be obtained by either casting a\nvalue to the :eql:type:`json` type, or by using the :eql:func:`to_json`\nfunction:\n\n.. code-block:: edgeql-repl\n\n db> select to_json('{\"hello\": \"world\"}');\n {Json(\"{\\\"hello\\\": \\\"world\\\"}\")}\n db> select 'hello world';\n {Json(\"\\\"hello world\\\"\")}\n\nAny value in Gel can be cast to a :eql:type:`json` type as well:\n\n.. code-block:: edgeql-repl\n\n db> select 2019;\n {Json(\"2019\")}\n db> select cal::to_local_date(datetime_current(), 'UTC');\n {Json(\"\\\"2022-11-21\\\"\")}\n\nThe :eql:func:`json_object_pack` function provides one more way to\nconstruct JSON. It constructs a JSON object from an array of key/value\ntuples:\n\n.. code-block:: edgeql-repl\n\n db> select json_object_pack({(\"hello\", \"world\")});\n {Json(\"{\\\"hello\\\": \\\"world\\\"}\")}\n\nAdditionally, any :eql:type:`Object` in Gel can be cast as a\n:eql:type:`json` type. This produces the same JSON value as the\nJSON-serialized result of that said object. Furthermore, this result will\nbe the same as the output of a :eql:stmt:`select expression `/\n :eql:stmt:`offset ``` which renders as this:\n:eql:stmt:`the select statement \n \n \n \n \n\n \n \n\n {render.button('Send Reset Email')}\n \n '''\n\n return render.base_page(\n title=f'Reset password{f\" for {app_name}\" if app_name else \"\"}',\n logo_url=logo_url,\n dark_logo_url=dark_logo_url,\n brand_color=brand_color,\n cleanup_search_params=['error', 'email', 'email_sent'],\n content=f'''\n {render.title('Reset password', join='for', app_name=app_name)}\n {content}\n {render.bottom_note(\"Back to\", link=\"Sign In\", href=\"signin\")}\n ''',\n )\n\n\ndef render_reset_password_page(\n *,\n base_path: str,\n provider_name: str,\n is_valid: bool,\n redirect_to: str,\n challenge: str,\n reset_token: Optional[str] = None,\n error_message: Optional[str] = None,\n is_code_flow: bool = False,\n email: Optional[str] = None,\n # config\n app_name: Optional[str] = None,\n logo_url: Optional[str] = None,\n dark_logo_url: Optional[str] = None,\n brand_color: Optional[str] = None,\n) -> bytes:\n if not is_valid:\n content = render.error_message(\n f'''Reset token is invalid, it may have expired.\n \n Try sending another reset email\n ''',\n False,\n )\n elif is_code_flow and email:\n content = f'''\n {render.error_message(error_message)}\n

We've sent a 6-digit reset code to {\n html.escape(email)\n }

\n {\n render.code_input_form(\n action=\"../reset-password\",\n email=email,\n provider=provider_name,\n label=\"Enter reset code\",\n button_text=\"Reset Password\",\n additional_fields=f'''\n \n \n \n \n \n ''',\n )\n }\n '''\n else:\n content = f'''\n {render.error_message(error_message)}\n\n
\n \n \n \n \n\n \n \n\n {render.button('Reset Password')}\n
'''\n\n return render.base_page(\n title=f'Reset password{f\" for {app_name}\" if app_name else \"\"}',\n logo_url=logo_url,\n dark_logo_url=dark_logo_url,\n brand_color=brand_color,\n cleanup_search_params=['error'],\n content=f'''\n {render.title('Reset password', join='for', app_name=app_name)}\n {content}\n ''',\n )\n\n\ndef render_email_verification_page_code_flow(\n *,\n email: str,\n provider: str,\n callback_url: str,\n base_path: str,\n challenge: str,\n error_message: Optional[str] = None,\n # config\n app_name: Optional[str] = None,\n logo_url: Optional[str] = None,\n dark_logo_url: Optional[str] = None,\n brand_color: Optional[str] = None,\n) -> bytes:\n \"\"\"Renders verification page that handles both link and code flows.\"\"\"\n\n content = f'''\n {render.error_message(error_message)}\n

We've sent a 6-digit verification code to {\n html.escape(email)\n }

\n {\n render.code_input_form(\n action=\"../verify\",\n email=email,\n provider=provider,\n label=\"Enter verification code\",\n button_text=\"Verify Email\",\n additional_fields=f'''\n \n \n \n '''\n )\n }\n '''\n\n return render.base_page(\n title=f'Verify email{f\" for {app_name}\" if app_name else \"\"}',\n logo_url=logo_url,\n dark_logo_url=dark_logo_url,\n brand_color=brand_color,\n cleanup_search_params=['error'],\n content=f'''\n {render.title('Verify email', join='for', app_name=app_name)}\n {content}\n ''',\n )\n\n\ndef render_email_verification_page_link_flow(\n *,\n is_valid: bool,\n error_messages: list[str],\n verification_token: Optional[str] = None,\n # config\n app_name: Optional[str] = None,\n logo_url: Optional[str] = None,\n dark_logo_url: Optional[str] = None,\n brand_color: Optional[str] = None,\n) -> bytes:\n resend_url = None\n if verification_token:\n verification_token = html.escape(verification_token)\n resend_url = (\n f\"resend-verification?verification_token={verification_token}\"\n )\n if not is_valid:\n messages = ''.join(\n [render.error_message(error) for error in error_messages]\n )\n content = f'''\n {messages}\n {\n (\n f'Try sending another verification'\n 'email'\n )\n if resend_url\n else ''\n }\n '''\n else:\n content = '''\n Email has been successfully verified. You may now\n sign in\n '''\n\n return render.base_page(\n title=f'Verify email{f\" for {app_name}\" if app_name else \"\"}',\n logo_url=logo_url,\n dark_logo_url=dark_logo_url,\n brand_color=brand_color,\n cleanup_search_params=['error'],\n content=f'''\n {render.title('Verify email', join='for', app_name=app_name)}\n {content}\n ''',\n )\n\n\ndef render_email_verification_expired_page(\n verification_token: str,\n # config\n app_name: Optional[str] = None,\n logo_url: Optional[str] = None,\n dark_logo_url: Optional[str] = None,\n brand_color: Optional[str] = None,\n) -> bytes:\n verification_token = html.escape(verification_token)\n content = render.error_message(\n f'''\n Your verification token has expired.\n \n Click here to resend the verification email\n \n ''',\n False,\n )\n\n return render.base_page(\n title=f'Verification expired{f\" for {app_name}\" if app_name else \"\"}',\n logo_url=logo_url,\n dark_logo_url=dark_logo_url,\n brand_color=brand_color,\n cleanup_search_params=['error'],\n content=f'''\n {\n render.title('Verification expired', join='for', app_name=app_name)\n }\n {content}\n ''',\n )\n\n\ndef render_resend_verification_done_page(\n *,\n is_valid: bool,\n verification_token: Optional[str] = None,\n # config\n app_name: Optional[str] = None,\n logo_url: Optional[str] = None,\n dark_logo_url: Optional[str] = None,\n brand_color: Optional[str] = None,\n) -> bytes:\n if verification_token is None:\n content = render.error_message(\n f\"\"\"\n Missing verification token, please follow the link provided in the\n original email, or on the signin page.\n \"\"\",\n False,\n )\n else:\n verification_token = html.escape(verification_token)\n if is_valid:\n content = f'''\n Your verification email has been resent. Please check your email.\n '''\n else:\n content = f'''\n Unable to resend verification email. Please try again.\n '''\n\n return render.base_page(\n title=(\n f'Email verification resent{f\" for {app_name}\" if app_name else \"\"}'\n ),\n logo_url=logo_url,\n dark_logo_url=dark_logo_url,\n brand_color=brand_color,\n cleanup_search_params=['error'],\n content=f'''\n {\n render.title(\n 'Email verification resent', join='for', app_name=app_name\n )\n }\n {content}\n ''',\n )\n\n\ndef render_magic_link_sent_page_code_flow(\n *,\n email: str,\n challenge: str,\n callback_url: str,\n error_message: Optional[str] = None,\n app_name: Optional[str] = None,\n logo_url: Optional[str] = None,\n dark_logo_url: Optional[str] = None,\n brand_color: Optional[str] = None,\n) -> bytes:\n content = f'''\n {render.error_message(error_message)}\n

We've sent a 6-digit sign-in code to {\n html.escape(email)\n }

\n {\n render.code_input_form(\n action=\"../magic-link/authenticate\",\n email=email,\n provider=\"builtin::local_magic_link\",\n label=\"Enter sign-in code\",\n button_text=\"Sign In\",\n additional_fields=f'''\n \n \n ''',\n )\n }\n '''\n title = f'Sign in code sent{f\" for {app_name}\" if app_name else \"\"}'\n page_title = 'Sign in code sent'\n\n return render.base_page(\n title=title,\n logo_url=logo_url,\n dark_logo_url=dark_logo_url,\n brand_color=brand_color,\n cleanup_search_params=['error'],\n content=f'''\n {render.title(page_title, join='for', app_name=app_name)}\n {content}\n ''',\n )\n\n\ndef render_magic_link_sent_page_link_flow(\n *,\n app_name: Optional[str] = None,\n logo_url: Optional[str] = None,\n dark_logo_url: Optional[str] = None,\n brand_color: Optional[str] = None,\n) -> bytes:\n content = render.success_message(\n \"A sign in link has been sent to your email. Please check your \"\n \"email.\"\n )\n title = f'Sign in link sent{f\" for {app_name}\" if app_name else \"\"}'\n page_title = 'Sign in link sent'\n\n return render.base_page(\n title=title,\n logo_url=logo_url,\n dark_logo_url=dark_logo_url,\n brand_color=brand_color,\n cleanup_search_params=['error'],\n content=f'''\n {render.title(page_title, join='for', app_name=app_name)}\n {content}\n ''',\n )\n\n\n# emails\n\n\ndef render_password_reset_email(\n *,\n to_addr: str,\n reset_url: str,\n app_name: Optional[str] = None,\n logo_url: Optional[str] = None,\n dark_logo_url: Optional[str] = None,\n brand_color: Optional[str] = render.DEFAULT_BRAND_COLOR,\n) -> email.message.EmailMessage:\n brand_color = brand_color or render.DEFAULT_BRAND_COLOR\n msg = email.message.EmailMessage()\n msg[\"To\"] = to_addr\n msg[\"Subject\"] = \"Reset password\"\n plain_text_content = f\"\"\"\nSomebody requested a new password for the {app_name or ''} account associated\nwith {to_addr}.\n\nPlease paste the following URL into your browser address bar to verify your\nemail address:\n\n{reset_url}\n \"\"\"\n html_content = f\"\"\"\n\n \n \n \n \n \n \n \n \n Somebody requested a new password for the {app_name or ''}\n account associated with {to_addr}.\n
\n \n \n \n \n \n No changes have been made to your account yet.\n
\n \n \n \n \n \n You can reset your password by clicking the button below:\n \n \n \n \n \n \n \n \n \n Reset your password\n \n \n \n \n \n \n \n \n \n In case the button didn't work, please paste the following URL\n into your browser address bar:\n

{reset_url}

\n \n \n \n \n \n \n If you did not request a new password, please let us know\n immediately by replying to this email.\n \n \n \n \n \n \n \n\n \"\"\" # noqa: E501\n\n msg[\"X-gel-password-reset-url\"] = reset_url\n msg.set_content(plain_text_content, subtype=\"plain\")\n msg.add_alternative(\n render.base_default_email(\n content=html_content,\n app_name=app_name,\n logo_url=logo_url,\n ),\n subtype=\"html\",\n )\n return msg\n\n\ndef render_verification_email(\n *,\n to_addr: str,\n verify_url: str,\n app_name: Optional[str] = None,\n logo_url: Optional[str] = None,\n dark_logo_url: Optional[str] = None,\n brand_color: Optional[str] = render.DEFAULT_BRAND_COLOR,\n) -> email.message.EmailMessage:\n brand_color = brand_color or render.DEFAULT_BRAND_COLOR\n msg = email.message.EmailMessage()\n msg[\"To\"] = to_addr\n msg[\"Subject\"] = (\n f\"Verify your email{f' for {app_name}' if app_name else ''}\"\n )\n plain_text_content = f\"\"\"\nCongratulations, you're registered{f' at {app_name}' if app_name else ''}!\n\nPlease paste the following URL into your browser address bar to verify your\nemail address:\n\n{verify_url}\n \"\"\"\n html_content = f\"\"\"\n\n \n \n Congratulations, you're registered\n {f'at {app_name}' if app_name else ''}!\n \n \n\n\n \n \n Please press the button below to verify your email address:\n \n \n\n\n \n \n \n \n \n Verify email address\n \n \n \n \n \n\n\n \n \n In case the button didn't work, please paste the following URL into\n your browser address bar:\n

{verify_url}

\n \n \n\n \"\"\"\n msg[\"X-gel-email-verify-url\"] = verify_url\n msg.set_content(plain_text_content, subtype=\"plain\")\n msg.add_alternative(\n render.base_default_email(\n content=html_content,\n app_name=app_name,\n logo_url=logo_url,\n ),\n subtype=\"html\",\n )\n return msg\n\n\ndef render_magic_link_email(\n *,\n to_addr: str,\n link: str,\n app_name: Optional[str] = None,\n logo_url: Optional[str] = None,\n dark_logo_url: Optional[str] = None,\n brand_color: Optional[str] = render.DEFAULT_BRAND_COLOR,\n) -> email.message.EmailMessage:\n brand_color = brand_color or render.DEFAULT_BRAND_COLOR\n msg = email.message.EmailMessage()\n msg[\"To\"] = to_addr\n msg[\"Subject\"] = f\"Sign in{f' to {app_name}' if app_name else ''}\"\n plain_text_content = f\"\"\"\nPlease paste the following URL into your browser address bar to be signed into\nyour account:\n\n{link}\n \"\"\"\n html_content = f\"\"\"\n\n \n \n Click the button below to sign in{f' to {app_name}' if app_name else ''}:\n \n \n\n\n \n \n \n \n \n Sign in\n \n \n \n \n \n\n\n \n \n In case the button didn't work, please paste the following URL\n into your browser address bar:\n

{link}

\n \n \n\n \"\"\" # noqa: E501\n\n msg[\"X-gel-magic-link\"] = link\n msg.set_content(plain_text_content, subtype=\"plain\")\n msg.add_alternative(\n render.base_default_email(\n content=html_content,\n app_name=app_name,\n logo_url=logo_url,\n ),\n subtype=\"html\",\n )\n return msg\n\n\ndef render_one_time_code_email(\n *,\n to_addr: str,\n code: str,\n app_name: Optional[str] = None,\n logo_url: Optional[str] = None,\n dark_logo_url: Optional[str] = None,\n brand_color: Optional[str] = render.DEFAULT_BRAND_COLOR,\n) -> email.message.EmailMessage:\n \"\"\"Renders an email containing a one-time verification code.\"\"\"\n brand_color = brand_color or render.DEFAULT_BRAND_COLOR\n msg = email.message.EmailMessage()\n msg[\"To\"] = to_addr\n msg[\"Subject\"] = (\n f\"Your verification code{f' for {app_name}' if app_name else ''}\"\n )\n plain_text_content = f\"\"\"\nYour verification code{f' for {app_name}' if app_name else ''} is:\n\n{code}\n\nThis code will expire in 10 minutes.\n \"\"\"\n html_content = f\"\"\"\n\n \n \n Your verification code{f' for {app_name}' if app_name else ''} is:\n \n \n\n\n \n \n {code}\n \n \n\n\n \n \n This code will expire in 10 minutes for your security.\n \n \n\n \"\"\" # noqa: E501\n\n msg[\"X-gel-email-verify-code\"] = code\n msg.set_content(plain_text_content, subtype=\"plain\")\n msg.add_alternative(\n render.base_default_email(\n content=html_content,\n app_name=app_name,\n logo_url=logo_url,\n ),\n subtype=\"html\",\n )\n return msg\n\n\ndef render_password_reset_code_email(\n *,\n to_addr: str,\n code: str,\n app_name: Optional[str] = None,\n logo_url: Optional[str] = None,\n dark_logo_url: Optional[str] = None,\n brand_color: Optional[str] = render.DEFAULT_BRAND_COLOR,\n) -> email.message.EmailMessage:\n \"\"\"Renders an email containing a one-time code for password reset.\"\"\"\n brand_color = brand_color or render.DEFAULT_BRAND_COLOR\n msg = email.message.EmailMessage()\n msg[\"To\"] = to_addr\n msg[\"Subject\"] = (\n f\"Password reset code{f' for {app_name}' if app_name else ''}\"\n )\n plain_text_content = f\"\"\"\nYour password reset code{f' for {app_name}' if app_name else ''} is:\n\n{code}\n\nThis code will expire in 10 minutes. If you didn't request a password reset, you can safely ignore this email.\n \"\"\" # noqa: E501\n html_content = f\"\"\"\n\n \n \n Your password reset code{f' for {app_name}' if app_name else ''} is:\n \n \n\n\n \n \n {code}\n \n \n\n\n \n \n This code will expire in 10 minutes for your security. If you didn't request a password reset, you can safely ignore this email.\n \n \n\n \"\"\" # noqa: E501\n\n msg[\"X-gel-password-reset-code\"] = code\n msg.set_content(plain_text_content, subtype=\"plain\")\n msg.add_alternative(\n render.base_default_email(\n content=html_content,\n app_name=app_name,\n logo_url=logo_url,\n ),\n subtype=\"html\",\n )\n return msg\n" }, { "path": "edb/server/protocol/auth_ext/ui/components.py", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2016-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\nfrom __future__ import annotations\nfrom typing import Optional, TYPE_CHECKING\n\nimport html\nimport re\nimport urllib.parse\n\nfrom . import util\n\nif TYPE_CHECKING:\n from edb.server.protocol.auth_ext import config as auth_config\n\nknown_oauth_provider_names = [\n 'builtin::oauth_github',\n 'builtin::oauth_google',\n 'builtin::oauth_apple',\n 'builtin::oauth_azure',\n 'builtin::oauth_discord',\n 'builtin::oauth_slack',\n]\n\n\nDEFAULT_BRAND_COLOR = \"1f8aed\"\n\n\ndef base_page(\n *,\n content: str,\n title: str,\n cleanup_search_params: list[str],\n logo_url: Optional[str] = None,\n dark_logo_url: Optional[str] = None,\n brand_color: Optional[str] = DEFAULT_BRAND_COLOR,\n) -> bytes:\n logo = ''\n if logo_url:\n logo = ''\n if dark_logo_url:\n logo += f''''''\n logo += f''\n\n cleanup_script = (\n f''''''\n if len(cleanup_search_params) > 0\n else ''\n )\n\n if (\n brand_color is None\n or util.hex_color_regexp.fullmatch(brand_color) is None\n ):\n brand_color = DEFAULT_BRAND_COLOR\n\n return f'''\n\n\n \n \n \n \n {html.escape(title)}\n {cleanup_script}\n \n \n \n {logo}\n
\n
\n {content}\n
\n
\n \n\n'''.encode()\n\n\ndef script(name: str) -> str:\n return f''\n\n\ndef title(title: str, *, app_name: Optional[str], join: str = 'to') -> str:\n if app_name is None:\n return f'''

{title}

'''\n\n return f'''

{title} {join} {html.escape(app_name)}

'''\n\n\ndef oauth_buttons(\n *,\n redirect_to: str,\n challenge: str,\n redirect_to_on_signup: Optional[str],\n oauth_providers: list[auth_config.OAuthProviderConfig],\n label_prefix: str,\n collapsed: bool,\n) -> str:\n if len(oauth_providers) == 0:\n return ''\n\n oauth_params = {\n 'redirect_to': redirect_to,\n 'challenge': challenge,\n }\n if redirect_to_on_signup:\n oauth_params['redirect_to_on_signup'] = redirect_to_on_signup\n\n buttons = '\\n'.join(\n [\n _oauth_button(p, oauth_params, label_prefix=label_prefix)\n for p in sorted(oauth_providers, key=lambda p: p.name)\n ]\n )\n\n return f'''\n
\n {buttons}\n
\n '''\n\n\ndef _oauth_button(\n provider: auth_config.OAuthProviderConfig,\n params: dict[str, str],\n *,\n label_prefix: str,\n) -> str:\n href = '../authorize?' + urllib.parse.urlencode(\n {'provider': provider.name, **params}\n )\n if (\n provider.name.startswith('builtin::')\n and provider.name in known_oauth_provider_names\n ):\n img = f'''\"{provider.display_name}'''\n elif provider.logo_url is not None:\n img = f'''\"{provider.display_name}'''\n else:\n img = ''\n\n label = f'{label_prefix} {provider.display_name}'\n return f'''\n \n {img}\n {label}\n \n '''\n\n\ndef button(\n text: Optional[str],\n *,\n id: Optional[str] = None,\n secondary: Optional[bool] = False,\n type: Optional[str] = 'submit',\n) -> str:\n classes = []\n if secondary:\n classes.append('secondary')\n if text is None:\n classes.append('icon-only')\n\n attrs = f'type=\"{type}\"'\n if id:\n attrs += f' id=\"{id}\"'\n if len(classes):\n attrs += f' class=\"{\" \".join(classes)}\"'\n\n return f'''\n '''\n\n\ndivider = '''\n
\n or\n
'''\n\n\ndef _slugify_label(label: str) -> str:\n slug = label.lower().strip()\n slug = re.sub(r\"[^a-z0-9]+\", \"-\", slug)\n slug = re.sub(r\"(^-|-$)\", \"\", slug)\n return slug or \"section\"\n\n\ndef tabs_content(\n sections: list[str], selected_tab: int, labels: Optional[list[str]] = None\n) -> str:\n content = ''\n\n for i, section in enumerate(sections):\n active = selected_tab == i\n aria_attrs = ''\n if labels is not None and i < len(labels):\n slug = _slugify_label(labels[i])\n aria_attrs = (\n ' role=\"tabpanel\" '\n f'id=\"panel-{slug}\" aria-labelledby=\"tab-{slug}\"'\n )\n hidden_attrs = ' aria-hidden=\"true\" hidden' if not active else ''\n else:\n hidden_attrs = '' if active else ''\n\n content += f'''\n \n {section}\n \n '''\n\n style = (\n f'style=\"transform: translateX({-100 * selected_tab}%)\"'\n if selected_tab > 0\n else ''\n )\n return f'''\n
\n {content}\n
\n '''\n\n\n_tab_underline = '''\n \n \n '''\n\n\ndef tabs_buttons(labels: list[str], selected_tab: int) -> str:\n content = ''\n\n for i, label in enumerate(labels):\n active = selected_tab == i\n slug = _slugify_label(label)\n aria_selected = 'true' if active else 'false'\n tabindex = '0' if active else '-1'\n content += f'''\n \n {label}\n {_tab_underline}\n \n '''\n\n return f'''\n
\n {content}\n
\n '''\n\n\ndef hidden_input(\n *, name: str, value: str, secondary_value: Optional[str] = None\n) -> str:\n return f''''''\n\n\ndef bottom_note(message: str, *, link: str, href: str) -> str:\n return f\"\"\"\n
\n {message}\n {link}\n
\n \"\"\"\n\n\ndef error_message(message: Optional[str], escape: bool = True) -> str:\n if message is None:\n return ''\n\n return f'''\n
\n \n \n \n {html.escape(message) if escape else message}\n
'''\n\n\ndef success_message(message: str) -> str:\n return f'''\n
\n \n \n \n \n {message}\n
\n '''\n\n\ndef code_input_form(\n *,\n action: str,\n email: str,\n provider: str,\n label: str = \"Enter verification code\",\n button_text: str = \"Verify Code\",\n additional_fields: str = \"\",\n) -> str:\n \"\"\"Renders a code input form with auto-formatting and mobile\n keyboard support.\"\"\"\n\n return f'''\n
\n \n \n \n \n\n {additional_fields}\n {button(button_text)}\n \n '''\n\n\ndef base_default_email(\n *,\n content: str,\n app_name: Optional[str],\n logo_url: Optional[str],\n) -> str:\n logo_html = (\n f\"\"\"\n \n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n \n \n \n \n \n \n \n\"\"\" # noqa: E501\n if logo_url\n else \"\"\n )\n\n return f\"\"\"\n\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n\n \n
\n{logo_html}\n
\n \n \n \n \n \n \n \n \n{content}\n \n \n
\n \n \n \n \n \n
\n \n \n \n\n\"\"\" # noqa: E501\n\n\n# Form Component Helpers\n# =====================\n\n\ndef get_magic_link_tab_label(verification_method: str) -> str:\n return \"Email Code\" if verification_method == \"Code\" else \"Email Link\"\n\n\ndef get_magic_link_button_text(verification_method: str) -> str:\n return (\n \"Email sign in code\"\n if verification_method == \"Code\"\n else \"Email sign in link\"\n )\n\n\ndef get_email_password_signup_redirect_url(\n verification_method: str, base_path: str, fallback_redirect: str\n) -> str:\n if verification_method == \"Code\":\n return f\"{base_path}/ui/verify?provider=builtin::local_emailpassword\"\n else:\n return fallback_redirect\n\n\ndef get_webauthn_signup_redirect_url(\n verification_method: str, base_path: str, fallback_redirect: str\n) -> str:\n if verification_method == \"Code\":\n return f\"{base_path}/ui/verify?provider=builtin::local_webauthn\"\n else:\n return fallback_redirect\n\n\ndef get_password_reset_redirect_url(\n verification_method: str, base_path: str, challenge: str\n) -> str:\n if verification_method == \"Code\":\n return f\"{base_path}/ui/reset-password\"\n else:\n return f\"{base_path}/ui/forgot-password?challenge={challenge}\"\n\n\ndef get_send_button_text(verification_method: str) -> str:\n return \"Send Code\" if verification_method == \"Code\" else \"Send Link\"\n\n\ndef get_verification_method_label(verification_method: str) -> str:\n return \"Email Code\" if verification_method == \"Code\" else \"Email Link\"\n\n\ndef render_base_email_form(\n *, id: str, challenge: str, email: str | None = None\n) -> str:\n return f\"\"\"\n \n \n \n \"\"\"\n\n\ndef render_password_input(\n *, challenge: str, should_show_forgot_password: bool\n) -> str:\n forgot_password_link = (\n f\"\"\"\n \n Forgot password?\n \n \"\"\"\n if should_show_forgot_password\n else ''\n )\n\n return f\"\"\"\n
\n \n {forgot_password_link}\n
\n \n \"\"\"\n\n\ndef render_password_form(\n *,\n challenge: str,\n email: str | None = None,\n redirect_to: str,\n base_path: str,\n provider_name: str,\n) -> str:\n return f\"\"\"\n \n \n \n \n {render_base_email_form(\n id=\"password-email\", challenge=challenge, email=email\n )}\n {render_password_input(\n challenge=challenge,\n should_show_forgot_password=True,\n )}\n {button(\"Sign In\", id=\"password-signin\")}\n \n \"\"\"\n\n\ndef render_webauthn_form(\n *,\n challenge: str,\n email: str | None = None,\n redirect_to: str,\n base_path: str,\n provider_name: str,\n) -> str:\n \"\"\"Render a complete WebAuthn authentication form.\"\"\"\n return f\"\"\"\n \n \n \n \n \n {render_base_email_form(\n id=\"webauthn-email\", challenge=challenge, email=email\n )}\n {button(\"Sign In\", id=\"webauthn-signin\")}\n \n \"\"\"\n\n\ndef render_magic_link_form(\n *,\n challenge: str,\n email: str | None = None,\n base_path: str,\n provider_name: str,\n callback_url: str | None = None,\n verification_method: str = \"Link\",\n) -> str:\n button_text = get_magic_link_button_text(verification_method)\n callback_field = (\n f'''\n \n '''\n if verification_method == \"Link\"\n else \"\"\n )\n\n return f\"\"\"\n \n \n \n \n {callback_field}\n {render_base_email_form(\n id=\"magic-link-email\", challenge=challenge, email=email\n )}\n {button(button_text, id=\"magic-link-signin\")}\n \n \"\"\"\n\n\n# Signup-specific form helpers\n# ===========================\n\n\ndef render_password_signup_form(\n *,\n challenge: str,\n email: str | None = None,\n redirect_to: str,\n base_path: str,\n provider_name: str,\n) -> str:\n return f\"\"\"\n \n \n \n \n \n {render_base_email_form(\n id=\"password-email\", challenge=challenge, email=email\n )}\n {render_password_input(\n challenge=challenge,\n should_show_forgot_password=False,\n )}\n {button(\"Sign Up\", id=\"password-signup\")}\n \n \"\"\"\n\n\ndef render_webauthn_signup_form(\n *,\n challenge: str,\n email: str | None = None,\n redirect_to: str,\n base_path: str,\n provider_name: str,\n) -> str:\n \"\"\"Render a complete WebAuthn signup form.\"\"\"\n return f\"\"\"\n \n \n \n \n \n \n {render_base_email_form(\n id=\"webauthn-email\", challenge=challenge, email=email\n )}\n {button(\"Sign Up\", id=\"webauthn-signup\")}\n \n \"\"\"\n\n\ndef render_magic_link_signup_form(\n *,\n challenge: str,\n email: str | None = None,\n base_path: str,\n provider_name: str,\n callback_url: str | None = None,\n verification_method: str = \"Link\",\n) -> str:\n \"\"\"Render a complete magic link/OTC signup form.\"\"\"\n tab_label = get_magic_link_tab_label(verification_method)\n callback_field = (\n f\"\"\"\n \n \"\"\"\n if verification_method == \"Link\"\n else \"\"\n )\n\n return f\"\"\"\n \n \n \n \n {callback_field}\n {render_base_email_form(\n id=\"magic-link-email\", challenge=challenge, email=email\n )}\n {button(f\"Sign Up with {tab_label}\", id=\"magic-link-signup\")}\n \n \"\"\"\n" }, { "path": "edb/server/protocol/auth_ext/ui/util.py", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2016-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\nimport re\n\n# Colour utils\n\nhex_color_regexp = re.compile(r'[0-9a-fA-F]{6}')\n\n\ndef get_colour_vars(bg_hex: str) -> str:\n bg_rgb = hex_to_rgb(bg_hex)\n bg_hsl = rgb_to_hsl(*bg_rgb)\n luma = rgb_to_luma(*bg_rgb)\n luma_dark = luma < 0.6\n\n text_color = hsl_to_rgb(\n bg_hsl[0], bg_hsl[1], min(90 if luma_dark else 35, bg_hsl[2])\n )\n dark_text_color = hsl_to_rgb(bg_hsl[0], bg_hsl[1], max(60, bg_hsl[2]))\n\n return f'''--accent-bg-color: #{bg_hex};\n --accent-bg-text-color: #{rgb_to_hex(\n *hsl_to_rgb(\n bg_hsl[0],\n bg_hsl[1],\n 95 if luma_dark else max(10, min(25, luma * 100 - 60))\n )\n )};\n --accent-bg-hover-color: #{rgb_to_hex(\n *hsl_to_rgb(\n bg_hsl[0], bg_hsl[1], bg_hsl[2] + (5 if luma_dark else -5)\n )\n )};\n --accent-text-color: #{rgb_to_hex(*text_color)};\n --accent-text-dark-color: #{rgb_to_hex(*dark_text_color)};\n --accent-focus-color: rgba({','.join(\n str(c) for c in text_color)},0.6);\n --accent-focus-dark-color: rgba({','.join(\n str(c) for c in dark_text_color)},0.6);'''\n\n\ndef hex_to_rgb(hex: str) -> tuple[float, float, float]:\n return (\n int(hex[0:2], base=16),\n int(hex[2:4], base=16),\n int(hex[4:6], base=16),\n )\n\n\ndef rgb_to_hex(r: float, g: float, b: float) -> str:\n return '%02x%02x%02x' % (int(r), int(g), int(b))\n\n\ndef rgb_to_luma(r: float, g: float, b: float) -> float:\n return (r * 0.299 + g * 0.587 + b * 0.114) / 255\n\n\ndef rgb_to_hsl(r: float, g: float, b: float) -> tuple[float, float, float]:\n r /= 255\n g /= 255\n b /= 255\n l = max(r, g, b)\n s = l - min(r, g, b)\n h = (\n (\n ((g - b) / s)\n if l == r\n else (2 + (b - r) / s) if l == g else (4 + (r - g) / s)\n )\n if s != 0\n else 0\n )\n return (\n 60 * h + 360 if 60 * h < 0 else 60 * h,\n 100\n * (\n (s / (2 * l - s) if l <= 0.5 else s / (2 - (2 * l - s)))\n if s != 0\n else 0\n ),\n (100 * (2 * l - s)) / 2,\n )\n\n\ndef hsl_to_rgb(h: float, s: float, l: float) -> tuple[float, float, float]:\n s /= 100\n l /= 100\n k = lambda n: (n + h / 30) % 12\n a = s * min(l, 1 - l)\n f = lambda n: l - a * max(-1, min(k(n) - 3, min(9 - k(n), 1)))\n return (\n round(255 * f(0)),\n round(255 * f(8)),\n round(255 * f(4)),\n )\n" }, { "path": "edb/server/protocol/auth_ext/util.py", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2016-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nfrom __future__ import annotations\n\nimport urllib.parse\nimport html\nimport logging\nimport asyncio\n\nfrom typing import (\n overload,\n Any,\n cast,\n Optional,\n TYPE_CHECKING,\n Callable,\n Awaitable,\n Mapping,\n)\n\nimport immutables\n\nfrom edb.common import retryloop\nfrom edb import errors as edb_errors\n\nfrom edb.server import config as edb_config, auth as jwt_auth\nfrom edb.server.config.types import CompositeConfigType\nfrom edb.server.protocol import execute\n\nfrom . import errors, config\n\nif TYPE_CHECKING:\n from edb.server import tenant as edbtenant\n from edb.server.dbview import dbview\n from edb.server import defines as edbdef\n\n\nlogger = logging.getLogger('edb.server.ext.auth')\n\n# Cache JWKSets for 10 minutes\njwtset_cache = jwt_auth.JWKSetCache(60 * 10)\n\n\nasync def json_query_no_retry(\n db: dbview.Database,\n query: str,\n *,\n variables: Mapping[str, Any] = immutables.Map(),\n tx_isolation: edbdef.TxIsolationLevel | None = None,\n role_name: str | None = None,\n) -> bytes:\n try:\n return await execute.parse_execute_json(\n db,\n query,\n variables=variables,\n tx_isolation=tx_isolation,\n role_name=role_name,\n cached_globally=True,\n query_tag='gel/auth',\n )\n except Exception as e:\n raise (await execute.interpret_error(e, db)) from None\n\n\nasync def json_query(\n db: dbview.Database,\n query: str,\n *,\n variables: Mapping[str, Any] = immutables.Map(),\n tx_isolation: edbdef.TxIsolationLevel | None = None,\n role_name: str | None = None,\n retry_timeout: float = 5.0,\n) -> bytes:\n # TODO: Should we move the retry into a function in execute instead?\n rloop = retryloop.RetryLoop(\n backoff=retryloop.exp_backoff(),\n timeout=retry_timeout,\n ignore=(edb_errors.TransactionConflictError,),\n )\n async for iteration in rloop:\n async with iteration:\n return await json_query_no_retry(\n db,\n query,\n variables=variables,\n tx_isolation=tx_isolation,\n role_name=role_name,\n )\n raise AssertionError('retryloop is broken')\n\n\ndef maybe_get_config_unchecked(db: edbtenant.dbview.Database, key: str) -> Any:\n return edb_config.lookup(key, db.db_config, spec=db.user_config_spec)\n\n\n@overload\ndef maybe_get_config[T](\n db: Any, key: str, expected_type: type[T]\n) -> T | None: ...\n\n\n@overload\ndef maybe_get_config(db: Any, key: str) -> str | None: ...\n\n\ndef maybe_get_config(\n db: Any, key: str, expected_type: type[object] = str\n) -> object:\n value = maybe_get_config_unchecked(db, key)\n\n if value is None:\n return None\n\n if not isinstance(value, expected_type):\n raise TypeError(\n f\"Config value `{key}` must be {expected_type.__name__}, got \"\n f\"{type(value).__name__}\"\n )\n\n return value\n\n\n@overload\ndef get_config[T](db: Any, key: str, expected_type: type[T]) -> T: ...\n\n\n@overload\ndef get_config(db: Any, key: str) -> str: ...\n\n\ndef get_config(db: Any, key: str, expected_type: type[object] = str) -> object:\n value = maybe_get_config(db, key, expected_type)\n if value is None:\n raise errors.MissingConfiguration(\n key=key,\n description=\"Missing configuration value\",\n )\n return value\n\n\ndef get_config_unchecked(db: Any, key: str) -> Any:\n value = maybe_get_config_unchecked(db, key)\n if value is None:\n raise errors.MissingConfiguration(\n key=key,\n description=\"Missing configuration value\",\n )\n return value\n\n\ndef get_config_typename(config_value: edb_config.SettingValue) -> str:\n return config_value._tspec.name # type: ignore\n\n\ndef escape_and_truncate(input_str: str | None, max_len: int) -> str | None:\n if input_str is None:\n return None\n trunc = (\n f\"{input_str[:max_len]}...\" if len(input_str) > max_len else input_str\n )\n return html.escape(trunc)\n\n\ndef get_app_details_config(db: Any) -> config.AppDetailsConfig:\n ui_config = cast(\n Optional[config.UIConfig],\n maybe_get_config(db, \"ext::auth::AuthConfig::ui\", CompositeConfigType),\n )\n\n return config.AppDetailsConfig(\n app_name=escape_and_truncate(\n maybe_get_config(db, \"ext::auth::AuthConfig::app_name\")\n or (ui_config.app_name if ui_config else None),\n 100,\n ),\n logo_url=escape_and_truncate(\n maybe_get_config(db, \"ext::auth::AuthConfig::logo_url\")\n or (ui_config.logo_url if ui_config else None),\n 2000,\n ),\n dark_logo_url=escape_and_truncate(\n maybe_get_config(db, \"ext::auth::AuthConfig::dark_logo_url\")\n or (ui_config.dark_logo_url if ui_config else None),\n 2000,\n ),\n brand_color=escape_and_truncate(\n maybe_get_config(db, \"ext::auth::AuthConfig::brand_color\")\n or (ui_config.brand_color if ui_config else None),\n 8,\n ),\n )\n\n\ndef join_url_params(url: str, params: dict[str, str]) -> str:\n parsed_url = urllib.parse.urlparse(url)\n query_params = {\n **urllib.parse.parse_qs(parsed_url.query),\n **{key: [val] for key, val in params.items()},\n }\n new_query_params = urllib.parse.urlencode(query_params, doseq=True)\n return parsed_url._replace(query=new_query_params).geturl()\n\n\nasync def get_remote_jwtset(\n url: str,\n fetch_lambda: Callable[[str], Awaitable[jwt_auth.JWKSet]],\n) -> jwt_auth.JWKSet:\n \"\"\"\n Get a JWKSet from the cache, or fetch it from the given URL if it's not in\n the cache.\n \"\"\"\n is_fresh, jwtset = jwtset_cache.get(url)\n match (is_fresh, jwtset):\n case (_, None):\n jwtset = await fetch_lambda(url)\n jwtset_cache.set(url, jwtset)\n case (True, jwtset):\n pass\n case _:\n # Run fetch in background to refresh cache\n async def refresh_cache(url: str) -> None:\n try:\n new_jwtset = await fetch_lambda(url)\n jwtset_cache.set(url, new_jwtset)\n except Exception:\n logger.exception(\n f\"Failed to refresh JWKSet cache for {url}\"\n )\n\n asyncio.create_task(refresh_cache(url))\n\n assert jwtset is not None\n return jwtset\n" }, { "path": "edb/server/protocol/auth_ext/webauthn.py", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2024-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\nfrom __future__ import annotations\n\nimport dataclasses\nimport base64\nimport json\nimport webauthn\n\nfrom typing import Optional, TYPE_CHECKING\nfrom webauthn.helpers import (\n parse_authentication_credential_json,\n structs as webauthn_structs,\n exceptions as webauthn_exceptions,\n)\n\nfrom edb.errors import ConstraintViolationError\n\nfrom . import config, data, errors, util, local\n\nif TYPE_CHECKING:\n from edb.server import tenant as edbtenant\n\n\n@dataclasses.dataclass(repr=False)\nclass WebAuthnRegistrationChallenge:\n \"\"\"\n Object that represents the ext::auth::WebAuthnRegistrationChallenge type\n \"\"\"\n\n id: str\n challenge: bytes\n user_handle: bytes\n email: str\n\n\nclass Client(local.Client):\n def __init__(self, db: edbtenant.dbview.Database):\n self.db = db\n self.provider = self._get_provider()\n self.app_name = self._get_app_name()\n self.config = self._get_provider_config(\"builtin::local_webauthn\")\n\n def _get_provider(self) -> config.WebAuthnProvider:\n provider_name = \"builtin::local_webauthn\"\n provider_client_config = util.get_config(\n self.db, \"ext::auth::AuthConfig::providers\", frozenset\n )\n for cfg in provider_client_config:\n if cfg.name == provider_name:\n return config.WebAuthnProvider(\n name=cfg.name,\n relying_party_origin=cfg.relying_party_origin,\n require_verification=cfg.require_verification,\n verification_method=cfg.verification_method,\n )\n\n raise errors.MissingConfiguration(\n provider_name, f\"Provider is not configured\"\n )\n\n def _get_app_name(self) -> Optional[str]:\n app_config = util.get_app_details_config(self.db)\n return app_config.app_name\n\n async def create_registration_options_for_email(\n self,\n email: str,\n ) -> tuple[str, bytes]:\n maybe_user_handle = await self._maybe_get_existing_user_handle(\n email=email\n )\n registration_options = webauthn.generate_registration_options(\n rp_id=self.provider.relying_party_id,\n rp_name=(self.app_name or self.provider.relying_party_origin),\n user_name=email,\n user_display_name=email,\n user_id=maybe_user_handle,\n )\n\n await self._create_registration_challenge(\n email=email,\n challenge=registration_options.challenge,\n user_handle=registration_options.user.id,\n )\n\n return (\n base64.urlsafe_b64encode(registration_options.user.id).decode(),\n webauthn.options_to_json(registration_options).encode(),\n )\n\n async def _maybe_get_existing_user_handle(\n self,\n email: str,\n ) -> Optional[bytes]:\n result = await util.json_query(\n self.db,\n \"\"\"\nwith\n email := $email,\n factors := (\n select ext::auth::WebAuthnFactor\n filter .email = email\n ),\nselect assert_single((select distinct factors.user_handle));\"\"\",\n variables={\n \"email\": email,\n },\n )\n\n result_json = json.loads(result.decode())\n if len(result_json) == 0:\n return None\n else:\n return base64.b64decode(result_json[0])\n\n async def _create_registration_challenge(\n self,\n email: str,\n challenge: bytes,\n user_handle: bytes,\n ) -> None:\n await util.json_query(\n self.db,\n \"\"\"\nwith\n challenge := $challenge,\n user_handle := $user_handle,\n email := $email,\ninsert ext::auth::WebAuthnRegistrationChallenge {\n challenge := challenge,\n user_handle := user_handle,\n email := email,\n}\"\"\",\n variables={\n \"challenge\": challenge,\n \"user_handle\": user_handle,\n \"email\": email,\n },\n )\n\n async def register(\n self,\n credentials: str,\n email: str,\n user_handle: bytes,\n ) -> data.EmailFactor:\n registration_challenge = await self._get_registration_challenge(\n email=email,\n user_handle=user_handle,\n )\n await self._delete_registration_challenges(\n email=email,\n user_handle=user_handle,\n )\n\n registration_verification = webauthn.verify_registration_response(\n credential=credentials,\n expected_challenge=registration_challenge.challenge,\n expected_rp_id=self.provider.relying_party_id,\n expected_origin=self.provider.relying_party_origin,\n )\n\n try:\n result = await util.json_query(\n self.db,\n \"\"\"\nwith\n email := $email,\n user_handle := $user_handle,\n credential_id := $credential_id,\n public_key := $public_key,\n identity := (insert ext::auth::LocalIdentity {\n issuer := \"local\",\n subject := \"\",\n }),\n factor := (insert ext::auth::WebAuthnFactor {\n email := email,\n user_handle := user_handle,\n credential_id := credential_id,\n public_key := public_key,\n identity := identity,\n }),\nselect factor { ** };\"\"\",\n variables={\n \"email\": email,\n \"user_handle\": user_handle,\n \"credential_id\": registration_verification.credential_id,\n \"public_key\": (\n registration_verification.credential_public_key\n ),\n },\n )\n except ConstraintViolationError:\n raise errors.UserAlreadyRegistered()\n\n result_json = json.loads(result.decode())\n assert len(result_json) == 1\n\n factor_dict = result_json[0]\n local_identity = data.LocalIdentity(**factor_dict.pop(\"identity\"))\n return data.WebAuthnFactor(**factor_dict, identity=local_identity)\n\n async def _get_registration_challenge(\n self,\n email: str,\n user_handle: bytes,\n ) -> WebAuthnRegistrationChallenge:\n result = await util.json_query(\n self.db,\n \"\"\"\nwith\n email := $email,\n user_handle := $user_handle,\nselect ext::auth::WebAuthnRegistrationChallenge {\n id,\n challenge,\n user_handle,\n email,\n}\nfilter .email = email and .user_handle = user_handle;\"\"\",\n variables={\n \"email\": email,\n \"user_handle\": user_handle,\n },\n )\n result_json = json.loads(result.decode())\n assert len(result_json) == 1\n challenge_dict = result_json[0]\n\n return WebAuthnRegistrationChallenge(\n id=challenge_dict[\"id\"],\n challenge=base64.b64decode(challenge_dict[\"challenge\"]),\n user_handle=base64.b64decode(challenge_dict[\"user_handle\"]),\n email=challenge_dict[\"email\"],\n )\n\n async def _delete_registration_challenges(\n self,\n email: str,\n user_handle: bytes,\n ) -> None:\n await util.json_query(\n self.db,\n \"\"\"\nwith\n email := $email,\n user_handle := $user_handle,\ndelete ext::auth::WebAuthnRegistrationChallenge\nfilter .email = email and .user_handle = user_handle;\"\"\",\n variables={\n \"email\": email,\n \"user_handle\": user_handle,\n },\n )\n\n async def create_authentication_options_for_email(\n self,\n *,\n webauthn_provider: config.WebAuthnProvider,\n email: str,\n ) -> tuple[str, bytes]:\n # Find credential IDs by email\n result = await util.json_query(\n self.db,\n \"\"\"\nselect ext::auth::WebAuthnFactor {\n user_handle,\n credential_id,\n}\nfilter .email = $email;\"\"\",\n variables={\n \"email\": email,\n },\n )\n result_json = json.loads(result.decode())\n if len(result_json) == 0:\n raise errors.WebAuthnAuthenticationFailed(\n \"No WebAuthn credentials found for this email.\"\n )\n\n user_handles: set[str] = {x[\"user_handle\"] for x in result_json}\n assert (\n len(user_handles) == 1\n ), \"Found WebAuthn multiple user handles for the same email.\"\n\n user_handle = base64.b64decode(result_json[0][\"user_handle\"])\n\n credential_ids = [\n webauthn_structs.PublicKeyCredentialDescriptor(\n base64.b64decode(x[\"credential_id\"])\n )\n for x in result_json\n ]\n\n registration_options = webauthn.generate_authentication_options(\n rp_id=webauthn_provider.relying_party_id,\n allow_credentials=credential_ids,\n )\n\n await util.json_query(\n self.db,\n \"\"\"\nwith\n challenge := $challenge,\n user_handle := $user_handle,\n email := $email,\n factors := (\n assert_exists((\n select ext::auth::WebAuthnFactor\n filter .user_handle = user_handle\n and .email = email\n ))\n )\ninsert ext::auth::WebAuthnAuthenticationChallenge {\n challenge := challenge,\n factors := factors,\n}\nunless conflict on .factors\nelse (\n update ext::auth::WebAuthnAuthenticationChallenge\n set {\n challenge := challenge\n }\n);\"\"\",\n variables={\n \"challenge\": registration_options.challenge,\n \"user_handle\": user_handle,\n \"email\": email,\n },\n )\n\n return (\n base64.urlsafe_b64encode(user_handle).decode(),\n webauthn.options_to_json(registration_options).encode(),\n )\n\n async def is_email_verified(\n self,\n email: str,\n assertion: str,\n ) -> bool:\n credential = parse_authentication_credential_json(assertion)\n\n result = await util.json_query(\n self.db,\n \"\"\"\nwith\n email := $email,\n credential_id := $credential_id,\n factor := assert_single((\n select ext::auth::WebAuthnFactor\n filter .email = email\n and credential_id = credential_id\n )),\nselect (factor.verified_at <= std::datetime_current()) ?? false;\"\"\",\n variables={\n \"email\": email,\n \"credential_id\": credential.raw_id,\n },\n )\n result_json = json.loads(result.decode())\n return bool(result_json[0])\n\n async def _get_authentication_challenge(\n self,\n email: str,\n credential_id: bytes,\n ) -> data.WebAuthnAuthenticationChallenge:\n result = await util.json_query(\n self.db,\n \"\"\"\nwith\n email := $email,\n credential_id := $credential_id,\nselect ext::auth::WebAuthnAuthenticationChallenge {\n id,\n created_at,\n modified_at,\n challenge,\n factors: {\n id,\n created_at,\n modified_at,\n email,\n verified_at,\n user_handle,\n credential_id,\n public_key,\n identity: {\n created_at,\n modified_at,\n id,\n issuer,\n subject,\n }\n },\n}\nfilter .factors.email = email and .factors.credential_id = credential_id;\"\"\",\n variables={\n \"email\": email,\n \"credential_id\": credential_id,\n },\n )\n result_json = json.loads(result.decode())\n if len(result_json) == 0:\n raise errors.WebAuthnAuthenticationFailed(\n \"Could not find a challenge. Please retry authentication.\"\n )\n elif len(result_json) > 1:\n raise errors.WebAuthnAuthenticationFailed(\n \"Multiple challenges found. Please retry authentication.\"\n )\n return data.WebAuthnAuthenticationChallenge(**result_json[0])\n\n async def _delete_authentication_challenges(\n self,\n email: str,\n credential_id: bytes,\n ) -> None:\n await util.json_query(\n self.db,\n \"\"\"\nwith\n email := $email,\n credential_id := $credential_id,\ndelete ext::auth::WebAuthnAuthenticationChallenge\nfilter .factors.email = email and .factors.credential_id = credential_id;\"\"\",\n variables={\n \"email\": email,\n \"credential_id\": credential_id,\n },\n )\n\n async def authenticate(\n self,\n *,\n email: str,\n assertion: str,\n ) -> data.LocalIdentity:\n credential = parse_authentication_credential_json(assertion)\n\n authentication_challenge = await self._get_authentication_challenge(\n email=email,\n credential_id=credential.raw_id,\n )\n await self._delete_authentication_challenges(\n email=email,\n credential_id=credential.raw_id,\n )\n\n factor = next(\n (\n f\n for f in authentication_challenge.factors\n if f.credential_id == credential.raw_id\n ),\n None,\n )\n assert factor is not None, \"Missing factor for the given credential.\"\n\n try:\n webauthn.verify_authentication_response(\n credential=credential,\n expected_challenge=authentication_challenge.challenge,\n credential_public_key=factor.public_key,\n credential_current_sign_count=0,\n expected_rp_id=self.provider.relying_party_id,\n expected_origin=self.provider.relying_party_origin,\n )\n except webauthn_exceptions.InvalidAuthenticationResponse:\n raise errors.WebAuthnAuthenticationFailed(\n \"Invalid authentication response. Please retry authentication.\"\n )\n\n return factor.identity\n\n async def get_email_factor_by_credential_id(\n self,\n credential_id: bytes,\n ) -> Optional[data.EmailFactor]:\n result = await util.json_query(\n self.db,\n \"\"\"\nwith\n credential_id := $credential_id,\nselect ext::auth::WebAuthnFactor {\n id,\n created_at,\n modified_at,\n email,\n verified_at,\n identity: {*},\n} filter .credential_id = credential_id;\"\"\",\n variables={\n \"credential_id\": credential_id,\n },\n )\n result_json = json.loads(result.decode())\n if len(result_json) == 0:\n return None\n elif len(result_json) > 1:\n # This should never happen given the exclusive constraint\n raise errors.WebAuthnAuthenticationFailed(\n \"Multiple WebAuthn factors found for the same credential ID.\"\n )\n return data.EmailFactor(**result_json[0])\n\n def _get_provider_config(\n self, provider_name: str\n ) -> config.WebAuthnProvider:\n provider_client_config = util.get_config(\n self.db, \"ext::auth::AuthConfig::providers\", frozenset\n )\n for cfg in provider_client_config:\n if cfg.name == provider_name:\n return config.WebAuthnProvider(\n name=cfg.name,\n relying_party_origin=cfg.relying_party_origin,\n require_verification=cfg.require_verification,\n verification_method=cfg.verification_method,\n )\n\n raise errors.MissingConfiguration(\n provider_name, f\"Provider is not configured\"\n )\n" }, { "path": "edb/server/protocol/auth_ext/webhook.py", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2024-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nfrom __future__ import annotations\n\nimport dataclasses\nimport typing\nimport abc\nimport datetime\nimport json\nimport hmac\nimport hashlib\nimport uuid\n\n\nfrom . import util\n\nif typing.TYPE_CHECKING:\n from edb.server import tenant as edbtenant\n\n\n@dataclasses.dataclass\nclass Event(abc.ABC):\n event_type: str\n event_id: str\n timestamp: datetime.datetime\n\n def __repr__(self) -> str:\n return (\n f\"{self.__class__.__name__}(\"\n f\"timestamp={self.timestamp!r}, \"\n f\"event_id={self.event_id!r}\"\n \")\"\n )\n\n\n@dataclasses.dataclass\nclass HasIdentity(abc.ABC):\n identity_id: str\n\n\n@dataclasses.dataclass\nclass HasEmailFactor(abc.ABC):\n email_factor_id: str\n\n\n@dataclasses.dataclass\nclass IdentityCreated(Event, HasIdentity):\n event_type: typing.Literal[\"IdentityCreated\"] = dataclasses.field(\n default=\"IdentityCreated\",\n init=False,\n )\n\n def __repr__(self) -> str:\n return (\n f\"{self.__class__.__name__}(\"\n f\"timestamp={self.timestamp}, \"\n f\"event_id={self.event_id}, \"\n f\"identity_id={self.identity_id}\"\n \")\"\n )\n\n\n@dataclasses.dataclass\nclass IdentityAuthenticated(Event, HasIdentity):\n event_type: typing.Literal[\"IdentityAuthenticated\"] = dataclasses.field(\n default=\"IdentityAuthenticated\",\n init=False,\n )\n\n def __repr__(self) -> str:\n return (\n f\"{self.__class__.__name__}(\"\n f\"timestamp={self.timestamp}, \"\n f\"event_id={self.event_id}, \"\n f\"identity_id={self.identity_id}\"\n \")\"\n )\n\n\n@dataclasses.dataclass\nclass EmailFactorCreated(Event, HasIdentity, HasEmailFactor):\n event_type: typing.Literal[\"EmailFactorCreated\"] = dataclasses.field(\n default=\"EmailFactorCreated\",\n init=False,\n )\n\n def __repr__(self) -> str:\n return (\n f\"{self.__class__.__name__}(\"\n f\"timestamp={self.timestamp}, \"\n f\"event_id={self.event_id}, \"\n f\"identity_id={self.identity_id}, \"\n f\"email_factor_id={self.email_factor_id}\"\n \")\"\n )\n\n\n@dataclasses.dataclass\nclass EmailVerificationRequested(Event, HasIdentity, HasEmailFactor):\n event_type: typing.Literal[\"EmailVerificationRequested\"] = (\n dataclasses.field(\n default=\"EmailVerificationRequested\",\n init=False,\n )\n )\n verification_token: str\n\n def __repr__(self) -> str:\n return (\n f\"{self.__class__.__name__}(\"\n f\"timestamp={self.timestamp}, \"\n f\"event_id={self.event_id}, \"\n f\"identity_id={self.identity_id}, \"\n f\"email_factor_id={self.email_factor_id}\"\n \")\"\n )\n\n\n@dataclasses.dataclass\nclass EmailVerified(Event, HasIdentity, HasEmailFactor):\n event_type: typing.Literal[\"EmailVerified\"] = dataclasses.field(\n default=\"EmailVerified\",\n init=False,\n )\n\n def __repr__(self) -> str:\n return (\n f\"{self.__class__.__name__}(\"\n f\"timestamp={self.timestamp}, \"\n f\"event_id={self.event_id}, \"\n f\"identity_id={self.identity_id}, \"\n f\"email_factor_id={self.email_factor_id}\"\n \")\"\n )\n\n\n@dataclasses.dataclass\nclass PasswordResetRequested(Event, HasIdentity, HasEmailFactor):\n event_type: typing.Literal[\"PasswordResetRequested\"] = dataclasses.field(\n default=\"PasswordResetRequested\",\n init=False,\n )\n reset_token: str\n\n def __repr__(self) -> str:\n return (\n f\"{self.__class__.__name__}(\"\n f\"timestamp={self.timestamp}, \"\n f\"event_id={self.event_id}, \"\n f\"identity_id={self.identity_id}, \"\n f\"email_factor_id={self.email_factor_id}\"\n \")\"\n )\n\n\n@dataclasses.dataclass\nclass MagicLinkRequested(Event, HasIdentity, HasEmailFactor):\n event_type: typing.Literal[\"MagicLinkRequested\"] = dataclasses.field(\n default=\"MagicLinkRequested\",\n init=False,\n )\n magic_link_token: str\n magic_link_url: str\n\n def __repr__(self) -> str:\n return (\n f\"{self.__class__.__name__}(\"\n f\"timestamp={self.timestamp}, \"\n f\"event_id={self.event_id}, \"\n f\"identity_id={self.identity_id}, \"\n f\"email_factor_id={self.email_factor_id}\"\n \")\"\n )\n\n\n@dataclasses.dataclass\nclass OneTimeCodeRequested(Event, HasIdentity, HasEmailFactor):\n event_type: typing.Literal[\"OneTimeCodeRequested\"] = dataclasses.field(\n default=\"OneTimeCodeRequested\",\n init=False,\n )\n otc_id: str\n one_time_code: str\n\n def __repr__(self) -> str:\n return (\n f\"{self.__class__.__name__}(\"\n f\"timestamp={self.timestamp}, \"\n f\"event_id={self.event_id}, \"\n f\"identity_id={self.identity_id}, \"\n f\"email_factor_id={self.email_factor_id}\"\n \")\"\n )\n\n\n@dataclasses.dataclass\nclass OneTimeCodeVerified(Event, HasIdentity, HasEmailFactor):\n event_type: typing.Literal[\"OneTimeCodeVerified\"] = dataclasses.field(\n default=\"OneTimeCodeVerified\",\n init=False,\n )\n otc_id: str\n\n def __repr__(self) -> str:\n return (\n f\"{self.__class__.__name__}(\"\n f\"timestamp={self.timestamp}, \"\n f\"event_id={self.event_id}, \"\n f\"identity_id={self.identity_id}, \"\n f\"email_factor_id={self.email_factor_id}\"\n \")\"\n )\n\n\nclass WebhookEncoder(json.JSONEncoder):\n def default(self, obj: typing.Any) -> typing.Any:\n if isinstance(obj, datetime.datetime):\n return obj.isoformat()\n if isinstance(obj, uuid.UUID):\n return str(obj)\n return super().default(obj)\n\n\nasync def send(\n db: edbtenant.dbview.Database,\n url: str,\n secret: typing.Optional[str],\n event: Event,\n) -> str:\n body = json.dumps(\n dataclasses.asdict(event), cls=WebhookEncoder\n ).encode()\n headers = [(\"Content-Type\", \"application/json\")]\n if secret:\n signature = hmac.new(\n secret.encode(), body, hashlib.sha256\n ).hexdigest()\n headers.append((\"x-ext-auth-signature-sha256\", signature))\n\n result_json = await util.json_query(\n db,\n \"\"\"\nwith\n nh as module std::net::http,\n net as module std::net,\n\n # n.b. workaround for bug in parse_execute_json\n url := $url,\n headers := >>$headers,\n body := $body,\n\n REQUEST := (\n nh::schedule_request(\n url,\n method := nh::Method.POST,\n headers := headers,\n body := body,\n )\n ),\nselect REQUEST;\n\"\"\",\n variables={\n \"url\": url,\n \"body\": body,\n \"headers\": headers,\n },\n )\n result = json.loads(result_json)\n\n match result[0][\"id\"]:\n case str(id):\n return id\n case _:\n raise ValueError(\n \"Expected single result with 'id' string property, got \"\n f\"{result[0]!r}\"\n )\n" }, { "path": "edb/server/protocol/auth_helpers.pxd", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2021-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\ncdef extract_token_from_auth_data(bytes auth_data)\ncdef auth_jwt(tenant, prefixed_token, str user, str dbname)\ncdef scram_get_verifier(tenant, str user)\ncdef parse_basic_auth(str auth_payload)\ncdef extract_http_user(scheme, auth_payload, params)\ncdef auth_basic(tenant, str username, str password)\ncdef auth_mtls(transport)\ncdef auth_mtls_with_user(transport, str username)\n" }, { "path": "edb/server/protocol/auth_helpers.pyx", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2019-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\"\"\"Authentication code that is shared between HTTP and binary protocols\"\"\"\n\nfrom edgedb import scram\n\nimport base64\nimport hashlib\nimport json\nimport logging\n\nfrom edb import errors\nfrom edb.server.auth import validate_gel_token\n\ncdef object logger = logging.getLogger('edb.server')\n\n\ncdef extract_token_from_auth_data(auth_data: bytes):\n header_value = auth_data.decode(\"ascii\")\n scheme, _, payload = header_value.partition(\" \")\n return scheme.lower(), payload.strip()\n\n\ncdef auth_jwt(tenant, prefixed_token: str | None, user: str, dbname: str):\n if not prefixed_token:\n raise errors.AuthenticationError(\n 'authentication failed: no authorization data provided')\n\n key = tenant.server.get_jws_key()\n if err := validate_gel_token(key, prefixed_token, user, dbname, tenant.get_instance_name()):\n raise errors.AuthenticationError(str(err))\n\n # Ensure it's a valid role, but check after the JWT is validated\n role = tenant.get_roles().get(user)\n if role is None:\n raise errors.AuthenticationError('authentication failed')\n\n\ncdef scram_get_verifier(tenant, user: str):\n roles = tenant.get_roles()\n\n rolerec = roles.get(user)\n if rolerec is not None:\n verifier_string = rolerec['password']\n if verifier_string is not None:\n try:\n verifier = scram.parse_verifier(verifier_string)\n except ValueError:\n raise errors.AuthenticationError(\n f'invalid SCRAM verifier for user {user!r}') from None\n is_mock = False\n return verifier, is_mock\n\n # To avoid revealing the validity of the submitted user name,\n # generate a mock verifier using a salt derived from the\n # received user name and the cluster mock auth nonce.\n # The same approach is taken by Postgres.\n nonce = tenant.get_instance_data('mock_auth_nonce')\n salt = hashlib.sha256(nonce.encode() + user.encode()).digest()\n\n verifier = scram.SCRAMVerifier(\n mechanism='SCRAM-SHA-256',\n iterations=scram.DEFAULT_ITERATIONS,\n salt=salt[:scram.DEFAULT_SALT_LENGTH],\n stored_key=b'',\n server_key=b'',\n )\n is_mock = True\n return verifier, is_mock\n\n\ndef scram_verify_password(password: str, verifier: object) -> bool:\n \"\"\"Check the given password against a verifier.\n\n Returns True if the password is OK, False otherwise.\n \"\"\"\n\n # adapted from edgedb-python's scram.verify_password but made to\n # take a verifier object instead of a string\n\n bpassword = scram.saslprep(password).encode('utf-8')\n salted_password = scram.get_salted_password(\n bpassword, verifier.salt, verifier.iterations)\n computed_key = scram.get_server_key(salted_password)\n return verifier.server_key == computed_key\n\n\ncdef parse_basic_auth(auth_payload: str):\n try:\n decoded = base64.b64decode(auth_payload).decode('utf-8')\n except ValueError:\n raise errors.AuthenticationError(\n 'authentication failed: malformed authentication') from None\n username, colon, password = decoded.partition(':')\n if colon != ':':\n raise errors.AuthenticationError(\n 'authentication failed: malformed authentication')\n return username, password\n\n\ncdef extract_http_user(scheme, auth_payload, params):\n \"\"\"Extract the username from an HTTP request.\n\n Raises an AuthenticationError if something is too malformed.\n\n Returns the username, along with the password, if appropriate.\n (To avoid needing to parse the packet twice.)\n \"\"\"\n\n if scheme == 'basic':\n return parse_basic_auth(auth_payload)\n else:\n # Respect X-EdgeDB-User if present, but otherwise default to 'edgedb'\n if params and b'user' in params:\n username = params[b'user'].decode('ascii')\n else:\n username = 'edgedb'\n return username, None\n\n\ncdef auth_basic(tenant, username: str, password: str):\n verifier, mock_auth = scram_get_verifier(tenant, username)\n if not scram_verify_password(password, verifier) or mock_auth:\n raise errors.AuthenticationError('authentication failed')\n\n\ncdef auth_mtls(transport):\n sslobj = transport.get_extra_info('ssl_object')\n if sslobj is None:\n raise errors.AuthenticationError(\n \"mTLS authentication is not supported over plaintext transport\")\n cert_data = sslobj.getpeercert()\n if not cert_data: # None or empty dict\n # If --tls-client-ca-file is specified, the SSLContext used here would\n # have done load_verify_locations() in `server/server.py`, and we will\n # have a valid client certificate (non-empty dict) now if one was\n # provided by the client and passed validation; empty dict otherwise.\n # `None` just means the peer didn't send a client certificate.\n raise errors.AuthenticationError(\n \"valid client certificate required\")\n return cert_data\n\n\ncdef auth_mtls_with_user(transport, str username):\n cert_data = auth_mtls(transport)\n try:\n for rdn in cert_data[\"subject\"]:\n if rdn[0][0] == 'commonName':\n if rdn[0][1] == username:\n return\n except Exception as ex:\n raise errors.AuthenticationError(\n \"bad client certificate\") from ex\n\n raise errors.AuthenticationError(\n f\"Common Name of client certificate doesn't match {username!r}\",\n )\n" }, { "path": "edb/server/protocol/binary.pxd", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2016-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\ncimport cython\ncimport cpython\n\nfrom libc.stdint cimport int8_t, uint8_t, int16_t, uint16_t, \\\n int32_t, uint32_t, int64_t, uint64_t\n\nfrom edb.server.pgproto.pgproto cimport (\n WriteBuffer,\n)\n\nfrom edb.server.dbview cimport dbview\nfrom edb.server.pgproto.debug cimport PG_DEBUG\nfrom edb.server.protocol cimport frontend\n\n\ncdef enum EdgeSeverity:\n EDGE_SEVERITY_DEBUG = 20\n EDGE_SEVERITY_INFO = 40\n EDGE_SEVERITY_NOTICE = 60\n EDGE_SEVERITY_WARNING = 80\n EDGE_SEVERITY_ERROR = 120\n EDGE_SEVERITY_FATAL = 200\n EDGE_SEVERITY_PANIC = 255\n\n\ncdef enum EdgeConnectionStatus:\n EDGECON_NEW = 0\n EDGECON_STARTED = 1\n EDGECON_OK = 2\n EDGECON_BAD = 3\n\n\ncdef class EdgeConnection(frontend.FrontendConnection):\n\n cdef:\n EdgeConnectionStatus _con_status\n\n readonly dbview.DatabaseConnectionView _dbview\n\n object _startup_msg_waiter\n\n dbview.CompiledQuery _last_anon_compiled\n int64_t _last_anon_compiled_hash\n\n bint query_cache_enabled\n\n tuple protocol_version\n tuple max_protocol\n tuple min_protocol\n\n object last_state\n int last_state_id\n\n bint _in_dump_restore\n\n bytes _auth_data\n dict _conn_params\n\n cdef inline dbview.DatabaseConnectionView get_dbview(self)\n\n cdef parse_execute_request(self)\n cdef parse_cardinality(self, bytes card)\n cdef char render_cardinality(self, query_unit) except -1\n\n cdef fallthrough(self)\n\n cdef sync_status(self)\n\n cdef WriteBuffer make_negotiate_protocol_version_msg(\n self, tuple target_proto\n )\n cdef WriteBuffer make_command_data_description_msg(\n self, dbview.CompiledQuery query\n )\n cdef WriteBuffer make_state_data_description_msg(self)\n cdef WriteBuffer make_command_complete_msg(self, capabilities, status)\n\n cdef inline ignore_headers(self)\n cdef dict parse_headers(self)\n cdef dict parse_annotations(self)\n cdef inline ignore_annotations(self)\n cdef get_checked_tag(self, dict annotations)\n\n cdef write_status(self, bytes name, bytes value)\n cdef write_edgedb_error(self, exc)\n\n cdef write_log(self, EdgeSeverity severity, uint32_t code, str message)\n\n\n@cython.final\ncdef class VirtualTransport:\n cdef:\n WriteBuffer buf\n bint closed\n object transport\n" }, { "path": "edb/server/protocol/binary.pyx", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2016-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\nimport asyncio\nimport base64\nimport collections\nimport contextlib\nimport json\nimport logging\nimport time\nimport statistics\nimport traceback\nimport sys\n\ncimport cython\ncimport cpython\n\nfrom typing import Dict, List, Optional, Sequence, Tuple\nfrom edb.server.protocol cimport cpythonx\n\nfrom libc.stdint cimport int8_t, uint8_t, int16_t, uint16_t, \\\n int32_t, uint32_t, int64_t, uint64_t, \\\n UINT32_MAX\n\nimport immutables\n\nfrom edb import buildmeta\nfrom edb import edgeql\nfrom edb.edgeql import qltypes\nfrom edb.graphql import tokenizer as gql_tokenizer\n\nfrom edb.pgsql import parser as pgparser\nfrom edb.graphql import tokenizer as gql_tokenizer\n\nfrom edb.server.pgproto cimport hton\nfrom edb.server.pgproto.pgproto cimport (\n WriteBuffer,\n ReadBuffer,\n\n FRBuffer,\n frb_init,\n frb_read,\n frb_read_all,\n frb_get_len,\n)\nfrom edb.server.pgproto.pgproto import UUID as pg_UUID\n\nfrom edb.server.dbview cimport dbview\n\nfrom edb.server import config\n\nfrom edb.server import args as srvargs\nfrom edb.server import compiler\nfrom edb.server import defines as edbdef\nfrom edb.server.compiler import errormech\nfrom edb.server.compiler import enums\nfrom edb.server.compiler import sertypes\nfrom edb.server.compiler cimport rpc\n\nfrom edb.server.protocol cimport auth_helpers\nfrom edb.server.protocol import execute\nfrom edb.server.protocol cimport frontend\nfrom edb.server.pgcon cimport pgcon\nfrom edb.server.pgcon import errors as pgerror\nfrom edb.server import metrics\n\nfrom edb.schema import objects as s_obj\n\nfrom edb import errors\nfrom edb.errors import base as base_errors, EdgeQLSyntaxError\nfrom edb.common import debug\n\nfrom edb.protocol import messages\n\n\nfrom edb import _graphql_rewrite\n\n\ninclude \"./consts.pxi\"\n\n\ncdef bytes EMPTY_TUPLE_UUID = s_obj.get_known_type_id('empty-tuple').bytes\n\ncdef uint64_t PROTO_CAPS = enums.Capability.PROTO_CAPS\n\ncdef object CARD_NO_RESULT = compiler.Cardinality.NO_RESULT\ncdef object CARD_AT_MOST_ONE = compiler.Cardinality.AT_MOST_ONE\ncdef object CARD_MANY = compiler.Cardinality.MANY\n\ncdef object FMT_NONE = compiler.OutputFormat.NONE\ncdef object FMT_BINARY = compiler.OutputFormat.BINARY\n\ncdef object LANG_EDGEQL = compiler.InputLanguage.EDGEQL\ncdef object LANG_SQL = compiler.InputLanguage.SQL\ncdef object LANG_GRAPHQL = compiler.InputLanguage.GRAPHQL\n\ncdef tuple DUMP_VER_MIN = (0, 7)\ncdef tuple DUMP_VER_MAX = edbdef.CURRENT_PROTOCOL\n\ncdef tuple MIN_PROTOCOL = edbdef.MIN_PROTOCOL\ncdef tuple CURRENT_PROTOCOL = edbdef.CURRENT_PROTOCOL\n\ncdef object logger = logging.getLogger('edb.server')\ncdef object log_metrics = logging.getLogger('edb.server.metrics')\n\nDEF QUERY_HEADER_DUMP_SECRETS = 0xFF10\n\n\ndef parse_catalog_version_header(value: bytes) -> uint64_t:\n if len(value) != 8:\n raise errors.BinaryProtocolError(\n f'catalog version value must be exactly 8 bytes (got {len(value)})'\n )\n cdef uint64_t catver = hton.unpack_uint64(cpython.PyBytes_AS_STRING(value))\n return catver\n\n\ncdef class EdgeConnection(frontend.FrontendConnection):\n interface = \"edgeql\"\n\n def __init__(\n self,\n server,\n tenant,\n *,\n auth_data: bytes,\n conn_params: dict[str, str] | None,\n protocol_version: edbdef.ProtocolVersion = CURRENT_PROTOCOL,\n **kwargs,\n ):\n super().__init__(server, tenant, **kwargs)\n self._con_status = EDGECON_NEW\n\n self._dbview = None\n\n self._last_anon_compiled = None\n\n self.query_cache_enabled = not (debug.flags.disable_qcache or\n debug.flags.edgeql_compile)\n\n self.protocol_version = protocol_version\n self.min_protocol = MIN_PROTOCOL\n self.max_protocol = CURRENT_PROTOCOL\n\n self._conn_params = conn_params\n\n self._in_dump_restore = False\n\n # Authentication data supplied by the transport (e.g. the content\n # of an HTTP Authorization header).\n self._auth_data = auth_data\n\n cdef is_in_tx(self):\n return self.get_dbview().in_tx()\n\n cdef inline dbview.DatabaseConnectionView get_dbview(self):\n if self._dbview is None:\n raise RuntimeError('Cannot access dbview while it is None')\n return self._dbview\n\n def debug_print(self, *args):\n if self._dbview is None:\n # This may happen before dbview is initialized, e.g. sending errors\n # to non-TLS clients due to mandatory TLS.\n print(\n '::EDGEPROTO::',\n f'id:{self._id}',\n f'in_tx:{0}',\n f'tx_error:{0}',\n *args,\n file=sys.stderr,\n )\n else:\n print(\n '::EDGEPROTO::',\n f'id:{self._id}',\n f'in_tx:{int(self._dbview.in_tx())}',\n f'tx_error:{int(self._dbview.in_tx_error())}',\n *args,\n file=sys.stderr,\n )\n\n def is_idle(self, expiry_time: float):\n # A connection is idle if it awaits for the next message for\n # client for too long (even if it is in an open transaction!)\n return (\n self._con_status != EDGECON_BAD and\n super().is_idle(expiry_time) and\n not self._in_dump_restore\n )\n\n def is_alive(self):\n return self._con_status == EDGECON_STARTED and super().is_alive()\n\n def close_for_idling(self):\n try:\n self.write_edgedb_error(\n errors.IdleSessionTimeoutError(\n 'closing the connection due to idling')\n )\n finally:\n self.close() # will flush\n\n cdef _after_idling(self):\n self.server.on_binary_client_after_idling(self)\n\n async def do_handshake(self):\n cdef:\n char mtype\n\n if self._transport_proto is srvargs.ServerConnTransport.HTTP:\n if self._conn_params is None:\n params = {}\n else:\n params = self._conn_params\n else:\n await self.wait_for_message(report_idling=True)\n mtype = self.buffer.get_message_type()\n if mtype != b'V':\n raise errors.BinaryProtocolError(\n f'unexpected initial message: \"{chr(mtype)}\", '\n f'expected \"V\"')\n\n params = await self._do_handshake()\n if self._conn_params is not None:\n params = self._conn_params + params\n\n return params\n\n async def auth(self, params):\n cdef:\n WriteBuffer msg_buf\n WriteBuffer buf\n\n user = params.get('user')\n if not user:\n raise errors.BinaryProtocolError(\n f'missing required connection parameter in ClientHandshake '\n f'message: \"user\"'\n )\n user = self.tenant.resolve_user_name(user)\n\n database = params.get('database')\n branch = params.get('branch')\n if not database and not branch:\n raise errors.BinaryProtocolError(\n f'missing required connection parameter in ClientHandshake '\n f'message: \"branch\" (or \"database\")'\n )\n database = self.tenant.resolve_branch_name(database, branch)\n\n logger.debug('received connection request by %s to database %s',\n user, database)\n\n await self._authenticate(user, database, params)\n\n logger.debug('successfully authenticated %s in database %s',\n user, database)\n\n if not self.tenant.is_database_connectable(database):\n raise errors.AccessError(\n f'database {database!r} does not accept connections'\n )\n\n self.dbname = database\n self.username = user\n # In the tunneled HTTP endpoint, auth gets done after we have\n # set up a dbview, so we need to update it..\n if self._dbview:\n self._dbview._role_name = user\n\n await self._start_connection(database)\n\n if self._transport_proto is srvargs.ServerConnTransport.HTTP:\n return\n\n buf = WriteBuffer()\n\n msg_buf = WriteBuffer.new_message(b'R')\n msg_buf.write_int32(0)\n msg_buf.end_message()\n buf.write_buffer(msg_buf)\n\n msg_buf = WriteBuffer.new_message(b'K')\n # TODO: should send ID of this connection\n msg_buf.write_bytes(b'\\x00' * 32)\n msg_buf.end_message()\n buf.write_buffer(msg_buf)\n\n if self.get_dbview().get_state_serializer() is None:\n await self.get_dbview().reload_state_serializer()\n buf.write_buffer(self.make_state_data_description_msg())\n\n self.write(buf)\n\n # In dev mode we expose the backend postgres DSN\n if self.server.in_dev_mode():\n params = self.tenant.get_pgaddr()\n params.update(database=self.tenant.get_pg_dbname(\n self.get_dbview().dbname\n ))\n params.clear_server_settings()\n self.write_status(b'pgdsn', params.to_dsn().encode())\n\n self.write_status(\n b'suggested_pool_concurrency',\n str(self.tenant.suggested_client_pool_size).encode()\n )\n self.write_status(\n b'system_config',\n self.tenant.get_report_config_data(self.protocol_version),\n )\n\n self.write(self.sync_status())\n\n self.flush()\n\n async def _do_handshake(self):\n cdef:\n uint16_t major\n uint16_t minor\n int i\n uint16_t reserved\n dict params = {}\n\n major = self.buffer.read_int16()\n minor = self.buffer.read_int16()\n\n self.protocol_version = major, minor\n\n nparams = self.buffer.read_int16()\n for i in range(nparams):\n k = self.buffer.read_len_prefixed_utf8()\n v = self.buffer.read_len_prefixed_utf8()\n params[k] = v\n\n reserved = self.buffer.read_int16()\n if reserved != 0:\n raise errors.BinaryProtocolError(\n f'unexpected value in reserved field of ClientHandshake')\n\n self.buffer.finish_message()\n\n negotiate = False\n if self.protocol_version < self.min_protocol:\n target_proto = self.min_protocol\n negotiate = True\n elif self.protocol_version > self.max_protocol:\n target_proto = self.max_protocol\n negotiate = True\n else:\n target_proto = self.protocol_version\n\n if negotiate:\n self.write(self.make_negotiate_protocol_version_msg(target_proto))\n self.flush()\n\n return params\n\n async def _start_connection(self, database: str) -> None:\n dbv = await self.tenant.new_dbview(\n dbname=database,\n query_cache=self.query_cache_enabled,\n protocol_version=self.protocol_version,\n role_name=self.username,\n )\n assert type(dbv) is dbview.DatabaseConnectionView\n self._dbview = dbv\n self.dbname = database\n\n self._con_status = EDGECON_STARTED\n\n cdef stop_connection(self):\n self._con_status = EDGECON_BAD\n\n if self._dbview is not None:\n self.tenant.remove_dbview(self._dbview)\n self._dbview = None\n\n def _auth_jwt(self, user, database, params):\n # token in the HTTP header has higher priority than\n # the ClientHandshake message, under the scenario of\n # binary protocol over HTTP\n if self._auth_data:\n scheme, prefixed_token = auth_helpers.extract_token_from_auth_data(\n self._auth_data)\n if scheme != 'bearer':\n raise errors.AuthenticationError(\n 'authentication failed: unrecognized authentication scheme')\n else:\n prefixed_token = params.get('secret_key')\n\n return auth_helpers.auth_jwt(\n self.tenant, prefixed_token, user, database)\n\n cdef WriteBuffer _make_authentication_sasl_initial(self, list methods):\n cdef WriteBuffer msg_buf\n msg_buf = WriteBuffer.new_message(b'R')\n msg_buf.write_int32(10)\n # Number of auth methods followed by a series\n # of zero-terminated strings identifying each method,\n # sorted in the order of server preference.\n msg_buf.write_int32(len(methods))\n for method in methods:\n msg_buf.write_len_prefixed_bytes(method)\n return msg_buf.end_message()\n\n cdef _expect_sasl_initial_response(self):\n mtype = self.buffer.get_message_type()\n if mtype != b'p':\n raise errors.BinaryProtocolError(\n f'expected SASL response, got message type {mtype}')\n selected_mech = self.buffer.read_len_prefixed_bytes()\n client_first = self.buffer.read_len_prefixed_bytes()\n self.buffer.finish_message()\n if not client_first:\n # The client didn't send the Client Initial Response\n # in SASLInitialResponse, this is an error.\n raise errors.BinaryProtocolError(\n f'client did not send the Client Initial Response '\n f'data in SASLInitialResponse')\n return selected_mech, client_first\n\n cdef WriteBuffer _make_authentication_sasl_msg(\n self, bytes data, bint final\n ):\n cdef WriteBuffer msg_buf\n msg_buf = WriteBuffer.new_message(b'R')\n if final:\n msg_buf.write_int32(12)\n else:\n msg_buf.write_int32(11)\n msg_buf.write_len_prefixed_bytes(data)\n return msg_buf.end_message()\n\n cdef bytes _expect_sasl_response(self):\n mtype = self.buffer.get_message_type()\n if mtype != b'r':\n raise errors.BinaryProtocolError(\n f'expected SASL response, got message type {mtype}')\n client_final = self.buffer.read_len_prefixed_bytes()\n self.buffer.finish_message()\n return client_final\n\n async def _execute_script(\n self,\n compiled: object,\n bind_args: bytes,\n *,\n query_req: Optional[rpc.CompilationRequest] = None,\n ):\n cdef:\n pgcon.PGConnection conn\n dbview.DatabaseConnectionView dbv\n\n if self._cancelled:\n raise ConnectionAbortedError\n\n dbv = self.get_dbview()\n async with self.with_pgcon() as conn:\n await execute.execute_script(\n conn,\n dbv,\n compiled,\n bind_args,\n fe_conn=self,\n query_req=query_req,\n )\n\n def _tokenize(\n self,\n eql: bytes,\n lang: enums.InputLanguage,\n ) -> edgeql.Source:\n text = eql.decode('utf-8')\n if lang is LANG_EDGEQL:\n if debug.flags.edgeql_disable_normalization:\n return edgeql.Source.from_string(text)\n else:\n return edgeql.NormalizedSource.from_string(text)\n elif lang is LANG_SQL:\n if debug.flags.edgeql_disable_normalization:\n return pgparser.Source.from_string(text)\n else:\n return pgparser.NormalizedSource.from_string(text)\n elif lang is LANG_GRAPHQL:\n if debug.flags.edgeql_disable_normalization:\n return gql_tokenizer.Source.from_string(text)\n else:\n try:\n return gql_tokenizer.NormalizedSource.from_string(text)\n except Exception:\n return gql_tokenizer.Source.from_string(text)\n else:\n raise errors.UnsupportedFeatureError(\n f\"unsupported input language: {lang}\")\n\n async def _suppress_tx_timeout(self):\n async with self.with_pgcon() as conn:\n await conn.sql_execute(b'''\n select pg_catalog.set_config(\n 'idle_in_transaction_session_timeout', '0', true)\n ''')\n\n async def _restore_tx_timeout(self, dbview.DatabaseConnectionView dbv):\n old_timeout = dbv.get_session_config().get(\n 'session_idle_transaction_timeout',\n )\n timeout = (\n 'NULL' if not old_timeout\n else repr(old_timeout.value.to_backend_str())\n )\n async with self.with_pgcon() as conn:\n await conn.sql_execute(f'''\n select pg_catalog.set_config(\n 'idle_in_transaction_session_timeout', {timeout}, true)\n '''.encode('utf-8'))\n\n async def _parse(\n self,\n rpc.CompilationRequest query_req,\n uint64_t allow_capabilities,\n tag=None,\n ) -> dbview.CompiledQuery:\n cdef dbview.DatabaseConnectionView dbv\n dbv = self.get_dbview()\n if self.debug:\n source = query_req.source\n text = source.text()\n self.debug_print('PARSE', text)\n self.debug_print(\n 'Cache key',\n source.cache_key(),\n f\"protocol_version={query_req.protocol_version}\",\n f\"input_language={query_req.input_language}\",\n f\"output_format={query_req.output_format}\",\n f\"expect_one={query_req.expect_one}\",\n f\"implicit_limit={query_req.implicit_limit}\",\n f\"inline_typeids={query_req.inline_typeids}\",\n f\"inline_typenames={query_req.inline_typenames}\",\n f\"inline_objectids={query_req.inline_objectids}\",\n f\"allow_capabilities={allow_capabilities}\",\n f\"modaliazes={dbv.get_modaliases()}\",\n f\"session_config={dbv.get_session_config()}\",\n )\n self.debug_print('Extra variables', source.variables(),\n 'after', source.first_extra())\n\n query_unit_group = dbv.lookup_compiled_query(query_req)\n if query_unit_group is None:\n # If we have to do a compile within a transaction, suppress\n # the idle_in_transaction_session_timeout.\n suppress_timeout = dbv.in_tx() and not dbv.in_tx_error()\n if suppress_timeout:\n await self._suppress_tx_timeout()\n try:\n if query_req.input_language is LANG_SQL:\n async with self.with_pgcon() as pg_conn:\n return await dbv.parse(\n query_req,\n allow_capabilities=allow_capabilities,\n pgcon=pg_conn,\n tag=tag,\n )\n else:\n return await dbv.parse(\n query_req,\n allow_capabilities=allow_capabilities,\n send_log_message=(\n lambda code, s: self.write_log(\n EdgeSeverity.EDGE_SEVERITY_NOTICE,\n code,\n s,\n )\n )\n )\n finally:\n if suppress_timeout:\n try:\n await self._restore_tx_timeout(dbv)\n except pgerror.BackendError as ex:\n # dbv.parse() for LANG_SQL can send a SQL\n # query, which can put the transaction in a\n # bad state if it fails. If we fail because of\n # that, swallow it.\n if (\n query_req.input_language is not LANG_SQL\n or not ex.code_is(\n pgerror.ERRCODE_IN_FAILED_SQL_TRANSACTION\n )\n ):\n raise\n else:\n return dbv.as_compiled(query_req, query_unit_group)\n\n cdef parse_cardinality(self, bytes card):\n if card[0] == CARD_MANY.value:\n return CARD_MANY\n elif card[0] == CARD_AT_MOST_ONE.value:\n return CARD_AT_MOST_ONE\n else:\n try:\n card_name = compiler.Cardinality(card[0]).name\n except ValueError:\n raise errors.BinaryProtocolError(\n f'unknown expected cardinality \"{repr(card)[2:-1]}\"')\n else:\n raise errors.BinaryProtocolError(\n f'cardinality {card_name} cannot be requested')\n\n cdef char render_cardinality(self, query_unit_group) except -1:\n return query_unit_group.cardinality.value\n\n cdef dict parse_headers(self):\n cdef:\n dict attrs\n uint16_t num_fields\n uint16_t key\n bytes value\n\n attrs = {}\n num_fields = self.buffer.read_int16()\n while num_fields:\n key = self.buffer.read_int16()\n value = self.buffer.read_len_prefixed_bytes()\n attrs[key] = value\n num_fields -= 1\n return attrs\n\n cdef inline ignore_headers(self):\n cdef:\n uint16_t num_fields\n\n num_fields = self.buffer.read_int16()\n while num_fields:\n self.buffer.read_int16()\n self.buffer.read_len_prefixed_bytes()\n num_fields -= 1\n\n cdef dict parse_annotations(self):\n cdef:\n dict annos\n uint16_t num_annos\n str name, value\n\n annos = {}\n num_annos = self.buffer.read_int16()\n while num_annos:\n name = self.buffer.read_len_prefixed_utf8()\n value = self.buffer.read_len_prefixed_utf8()\n annos[name] = value\n num_annos -= 1\n return annos\n\n cdef inline ignore_annotations(self):\n cdef:\n uint16_t num_annos\n\n num_annos = self.buffer.read_int16()\n while num_annos:\n self.buffer.read_len_prefixed_bytes()\n self.buffer.read_len_prefixed_bytes()\n num_annos -= 1\n\n #############\n\n cdef WriteBuffer make_negotiate_protocol_version_msg(\n self,\n tuple target_proto,\n ):\n cdef:\n WriteBuffer msg\n\n # NegotiateProtocolVersion\n msg = WriteBuffer.new_message(b'v')\n # Highest supported major version of the protocol.\n msg.write_int16(target_proto[0])\n # Highest supported minor version of the protocol.\n msg.write_int16(target_proto[1])\n # No extensions are currently supported.\n msg.write_int16(0)\n\n msg.end_message()\n return msg\n\n cdef WriteBuffer make_command_data_description_msg(\n self, dbview.CompiledQuery query\n ):\n cdef:\n WriteBuffer msg\n int16_t ann_count\n\n msg = WriteBuffer.new_message(b'T')\n ann_count = 0\n if query.query_unit_group.warnings:\n ann_count += 1\n if query.query_unit_group.unsafe_isolation_dangers:\n ann_count += 1\n\n msg.write_int16(ann_count)\n\n if query.query_unit_group.warnings:\n warnings = json.dumps(\n [w.to_json() for w in query.query_unit_group.warnings]\n ).encode('utf-8')\n msg.write_len_prefixed_bytes(b'warnings')\n msg.write_len_prefixed_bytes(warnings)\n if query.query_unit_group.unsafe_isolation_dangers:\n dangers = json.dumps([\n w.to_json() for\n w in query.query_unit_group.unsafe_isolation_dangers\n ]).encode('utf-8')\n msg.write_len_prefixed_bytes(b'unsafe_isolation_dangers')\n msg.write_len_prefixed_bytes(dangers)\n\n msg.write_int64(\n query.query_unit_group.capabilities & PROTO_CAPS\n )\n msg.write_byte(self.render_cardinality(query.query_unit_group))\n\n in_data = query.query_unit_group.in_type_data\n msg.write_bytes(query.query_unit_group.in_type_id)\n msg.write_len_prefixed_bytes(in_data)\n\n out_data = query.query_unit_group.out_type_data\n msg.write_bytes(query.query_unit_group.out_type_id)\n msg.write_len_prefixed_bytes(out_data)\n\n msg.end_message()\n return msg\n\n cdef WriteBuffer make_state_data_description_msg(self):\n cdef WriteBuffer msg\n\n type_id, type_data = self.get_dbview().describe_state()\n\n msg = WriteBuffer.new_message(b's')\n msg.write_bytes(type_id.bytes)\n msg.write_len_prefixed_bytes(type_data)\n msg.end_message()\n\n return msg\n\n cdef WriteBuffer make_command_complete_msg(self, capabilities, status):\n cdef:\n WriteBuffer msg\n\n state_tid, state_data = self.get_dbview().encode_state()\n\n msg = WriteBuffer.new_message(b'C')\n msg.write_int16(0) # no annotations\n msg.write_int64(capabilities & PROTO_CAPS)\n msg.write_len_prefixed_bytes(status)\n\n msg.write_bytes(state_tid.bytes)\n msg.write_len_prefixed_bytes(state_data)\n\n return msg.end_message()\n\n async def _execute_rollback(self, compiled: dbview.CompiledQuery):\n cdef:\n dbview.DatabaseConnectionView _dbview\n pgcon.PGConnection conn\n\n query_unit = compiled.query_unit_group[0]\n _dbview = self.get_dbview()\n if not (\n query_unit.tx_savepoint_rollback or\n query_unit.tx_rollback or\n query_unit.tx_abort_migration\n ):\n _dbview.raise_in_tx_error()\n\n async with self.with_pgcon() as conn:\n if query_unit.sql:\n await conn.sql_execute(query_unit.sql)\n\n if query_unit.tx_abort_migration:\n _dbview.clear_tx_error()\n elif query_unit.tx_savepoint_rollback:\n _dbview.rollback_tx_to_savepoint(query_unit.sp_name)\n else:\n assert query_unit.tx_rollback\n _dbview.abort_tx()\n\n async def _execute(\n self,\n compiled: dbview.CompiledQuery,\n bind_args: bytes,\n use_prep_stmt: bint,\n *,\n query_req: Optional[rpc.CompilationRequest] = None,\n ):\n cdef:\n dbview.DatabaseConnectionView dbv\n pgcon.PGConnection conn\n\n dbv = self.get_dbview()\n async with self.with_pgcon() as conn:\n await execute.execute(\n conn,\n dbv,\n compiled,\n bind_args,\n fe_conn=self,\n use_prep_stmt=use_prep_stmt,\n query_req=query_req,\n )\n\n query_unit = compiled.query_unit_group[0]\n if query_unit.config_requires_restart:\n self.write_log(\n EdgeSeverity.EDGE_SEVERITY_NOTICE,\n errors.LogMessage.get_code(),\n 'server restart is required for the configuration '\n 'change to take effect')\n\n cdef parse_execute_request(self):\n cdef:\n uint64_t allow_capabilities = 0\n uint64_t compilation_flags = 0\n int64_t implicit_limit = 0\n bint inline_typenames = False\n bint inline_typeids = False\n bint inline_objectids = False\n object cardinality\n object output_format\n bint expect_one = False\n bytes query\n dbview.DatabaseConnectionView _dbview\n\n allow_capabilities = PROTO_CAPS & self.buffer.read_int64()\n compilation_flags = self.buffer.read_int64()\n implicit_limit = self.buffer.read_int64()\n\n if implicit_limit < 0:\n raise errors.BinaryProtocolError(\n f'implicit limit cannot be negative'\n )\n\n inline_typenames = (\n compilation_flags\n & messages.CompilationFlag.INJECT_OUTPUT_TYPE_NAMES\n )\n inline_typeids = (\n compilation_flags\n & messages.CompilationFlag.INJECT_OUTPUT_TYPE_IDS\n )\n inline_objectids = (\n compilation_flags\n & messages.CompilationFlag.INJECT_OUTPUT_OBJECT_IDS\n )\n\n if self.protocol_version >= (3, 0):\n lang = rpc.deserialize_input_language(self.buffer.read_byte())\n else:\n lang = LANG_EDGEQL\n\n output_format = rpc.deserialize_output_format(self.buffer.read_byte())\n if (\n lang is LANG_SQL\n and output_format is not FMT_NONE\n and output_format is not FMT_BINARY\n ):\n raise errors.UnsupportedFeatureError(\n \"non-binary output format is not supported with \"\n \"SQL as the input language\"\n )\n\n cardinality = self.parse_cardinality(self.buffer.read_byte())\n expect_one = cardinality is CARD_AT_MOST_ONE\n if lang is LANG_SQL and cardinality is not CARD_MANY:\n raise errors.UnsupportedFeatureError(\n \"output cardinality assertions are not supported with \"\n \"SQL as the input language\"\n )\n\n query = self.buffer.read_len_prefixed_bytes()\n if not query:\n raise errors.BinaryProtocolError('empty query')\n\n metrics.query_size.observe(\n len(query), self.get_tenant_label(), 'edgeql'\n )\n\n _dbview = self.get_dbview()\n state_tid = self.buffer.read_bytes(16)\n state_data = self.buffer.read_len_prefixed_bytes()\n try:\n _dbview.decode_state(state_tid, state_data)\n except errors.StateMismatchError:\n self.write(self.make_state_data_description_msg())\n raise\n\n cfg_ser = self.server.compilation_config_serializer\n rv = rpc.CompilationRequest(\n source=self._tokenize(query, lang),\n protocol_version=self.protocol_version,\n schema_version=_dbview.schema_version,\n compilation_config_serializer=cfg_ser,\n input_language=lang,\n output_format=output_format,\n expect_one=expect_one,\n implicit_limit=implicit_limit,\n inline_typeids=inline_typeids,\n inline_typenames=inline_typenames,\n inline_objectids=inline_objectids,\n modaliases=_dbview.get_modaliases(),\n session_config=_dbview.get_session_config(),\n database_config=_dbview.get_database_config(),\n system_config=_dbview.get_compilation_system_config(),\n role_name=self.username,\n branch_name=self.dbname,\n )\n return rv, allow_capabilities\n\n cdef get_checked_tag(self, dict annotations):\n tag = annotations.get(\"tag\")\n if not tag:\n return None\n if len(tag) > 128:\n raise errors.BinaryProtocolError(\n 'bad annotation: tag too long (> 128 bytes)')\n return tag\n\n async def parse(self):\n cdef:\n bytes eql\n rpc.CompilationRequest query_req\n dbview.DatabaseConnectionView _dbview\n WriteBuffer parse_complete\n WriteBuffer buf\n uint64_t allow_capabilities\n\n self._last_anon_compiled = None\n\n if self.protocol_version >= (3, 0):\n self.ignore_annotations()\n else:\n self.ignore_headers()\n\n _dbview = self.get_dbview()\n if _dbview.get_state_serializer() is None:\n await _dbview.reload_state_serializer()\n query_req, allow_capabilities = self.parse_execute_request()\n compiled = await self._parse(query_req, allow_capabilities)\n\n buf = self.make_command_data_description_msg(compiled)\n\n # Cache compilation result in anticipation that the client\n # will follow up with an Execute immediately.\n #\n # N.B.: we cannot rely on query cache because not all units\n # are cacheable.\n self._last_anon_compiled = compiled\n self._last_anon_compiled_hash = hash(query_req)\n\n self.write(buf)\n self.flush()\n\n async def execute(self):\n cdef:\n rpc.CompilationRequest query_req\n dbview.DatabaseConnectionView _dbview\n bytes in_tid\n bytes out_tid\n bytes args\n uint64_t allow_capabilities\n\n if self.protocol_version >= (3, 0):\n tag = self.get_checked_tag(self.parse_annotations())\n else:\n self.ignore_headers()\n tag = None\n\n _dbview = self.get_dbview()\n if _dbview.get_state_serializer() is None:\n await _dbview.reload_state_serializer()\n query_req, allow_capabilities = self.parse_execute_request()\n in_tid = self.buffer.read_bytes(16)\n out_tid = self.buffer.read_bytes(16)\n args = self.buffer.read_len_prefixed_bytes()\n self.buffer.finish_message()\n\n compiled = None\n if (\n self._last_anon_compiled is not None and\n hash(query_req) == self._last_anon_compiled_hash and\n in_tid == self._last_anon_compiled.query_unit_group.in_type_id and\n out_tid == self._last_anon_compiled.query_unit_group.out_type_id\n ):\n compiled = self._last_anon_compiled\n query_unit_group = compiled.query_unit_group\n else:\n query_unit_group = _dbview.lookup_compiled_query(query_req)\n\n if query_unit_group is None:\n if self.debug:\n self.debug_print('EXECUTE /CACHE MISS', query_req.source.text())\n\n compiled = await self._parse(query_req, allow_capabilities, tag)\n query_unit_group = compiled.query_unit_group\n\n # If this is a graphql request, and the compilation of it\n # depends on reading the value of some variables (because they\n # are used in @include or as params to type introspection, for\n # example) we need to reflect those variables into the\n # query_req and look it up again, and then maybe compile again.\n #\n # What a pain!\n if query_unit_group.graphql_key_variables:\n key_vars = _extract_key_vars(query_unit_group, query_req, args)\n query_req = query_req.__copy__()\n query_req.set_key_params(key_vars)\n\n compiled = None\n query_unit_group = _dbview.lookup_compiled_query(query_req)\n\n # If we had to do a graphql_key_variables lookup, we might need\n # to compile again.\n if query_unit_group is None:\n if self.debug:\n self.debug_print(\n 'EXECUTE /CACHE MISS (graphql nonsense)',\n query_req.source.text(),\n )\n\n compiled = await self._parse(query_req, allow_capabilities, tag)\n query_unit_group = compiled.query_unit_group\n else:\n if not compiled:\n compiled = _dbview.as_compiled(query_req, query_unit_group)\n\n compiled.tag = tag\n\n if self._cancelled:\n raise ConnectionAbortedError\n\n self._query_count += 1\n\n # Clear the _last_anon_compiled so that the next Execute - if\n # identical - will always lookup in the cache and honor the\n # `cacheable` flag to compile the query again.\n self._last_anon_compiled = None\n\n _dbview.check_capabilities(\n query_unit_group,\n allow_capabilities,\n errors.DisabledCapabilityError,\n \"disabled by the client\",\n unsafe_isolation_dangers=query_unit_group.unsafe_isolation_dangers,\n )\n\n if query_unit_group.in_type_id != in_tid:\n self.write(self.make_command_data_description_msg(compiled))\n raise errors.ParameterTypeMismatchError(\n \"specified parameter type(s) do not match the parameter \"\n \"types inferred from specified command(s)\"\n )\n\n if (\n query_unit_group.out_type_id != out_tid\n or query_unit_group.warnings\n ):\n # The client has no up-to-date information about the output,\n # so provide one.\n self.write(self.make_command_data_description_msg(compiled))\n\n if self.debug:\n self.debug_print('EXECUTE', query_req.source.text())\n\n force_script = any(x.needs_readback for x in query_unit_group)\n if (\n _dbview.in_tx_error()\n or query_unit_group[0].tx_savepoint_rollback\n or query_unit_group[0].tx_abort_migration\n ):\n assert len(query_unit_group) == 1\n await self._execute_rollback(compiled)\n elif len(query_unit_group) > 1 or force_script:\n await self._execute_script(compiled, args, query_req=query_req)\n else:\n use_prep = (\n len(query_unit_group) == 1\n and bool(query_unit_group[0].sql_hash)\n )\n await self._execute(compiled, args, use_prep, query_req=query_req)\n\n if self._cancelled:\n raise ConnectionAbortedError\n\n if _dbview.is_state_desc_changed():\n self.write(self.make_state_data_description_msg())\n self.write(\n self.make_command_complete_msg(\n compiled.query_unit_group.capabilities,\n compiled.query_unit_group[-1].status,\n )\n )\n self.flush()\n\n async def sync(self):\n self.buffer.consume_message()\n self.write(self.sync_status())\n\n if self.debug:\n self.debug_print('SYNC')\n\n self.flush()\n\n def check_readiness(self):\n if self.tenant.is_blocked():\n readiness_reason = self.tenant.get_readiness_reason()\n msg = \"the server is not accepting requests\"\n if readiness_reason:\n msg = f\"{msg}: {readiness_reason}\"\n raise errors.ServerBlockedError(msg)\n elif not self.tenant.is_online():\n readiness_reason = self.tenant.get_readiness_reason()\n msg = \"the server is going offline\"\n if readiness_reason:\n msg = f\"{msg}: {readiness_reason}\"\n raise errors.ServerOfflineError(msg)\n\n async def authenticate(self):\n self.check_readiness()\n params = await self.do_handshake()\n await self.auth(params)\n self.server.on_binary_client_authed(self)\n\n async def main_step(self, char mtype):\n try:\n self.check_readiness()\n\n if mtype == b'O':\n await self.execute()\n\n elif mtype == b'P':\n await self.parse()\n\n elif mtype == b'S':\n await self.sync()\n\n elif mtype == b'X':\n self.close()\n return True\n\n elif mtype == b'>':\n await self.dump()\n\n elif mtype == b'<':\n # The restore protocol cannot send SYNC beforehand,\n # so if an error occurs the server should send an\n # ERROR message immediately.\n await self.restore()\n\n elif mtype == b'D':\n raise errors.BinaryProtocolError(\n \"Describe message (D) is not supported in \"\n \"protocol versions greater than 0.13\")\n\n elif mtype == b'E':\n raise errors.BinaryProtocolError(\n \"Legacy Execute message (E) is not supported in \"\n \"protocol versions greater than 0.13\")\n\n elif mtype == b'Q':\n raise errors.BinaryProtocolError(\n \"ExecuteScript message (Q) is not supported in \"\n \"protocol versions greater then 0.13\")\n\n else:\n self.fallthrough()\n\n except ConnectionError:\n raise\n\n except asyncio.CancelledError:\n raise\n\n except Exception as ex:\n if self._cancelled and \\\n isinstance(ex, pgerror.BackendQueryCancelledError):\n # If we are cancelling the protocol (means that the\n # client side of the connection has dropped and we\n # need to gracefull cleanup and abort) we want to\n # propagate the BackendQueryCancelledError exception.\n #\n # If we're not cancelling, we'll treat it just like\n # any other error coming from Postgres (a query\n # might get cancelled due to a variety of reasons.)\n raise\n\n # The connection has been aborted; there's nothing\n # we can do except shutting this down.\n if self._con_status == EDGECON_BAD:\n return True\n\n self.get_dbview().tx_error()\n self.buffer.finish_message()\n\n ex = await self.interpret_error(ex)\n\n self.write_edgedb_error(ex)\n\n if isinstance(\n ex,\n (errors.ServerOfflineError, errors.ServerBlockedError),\n ):\n # This server is going into \"offline\" or \"blocked\" mode,\n # close the connection.\n self.write(self.sync_status())\n self.flush()\n self.close()\n return\n\n self.flush()\n\n # The connection was aborted while we were\n # interpreting the error (via compiler/errmech.py).\n if self._con_status == EDGECON_BAD:\n return True\n\n await self.recover_from_error()\n\n else:\n self.buffer.finish_message()\n\n cdef _main_task_stopped_normally(self):\n self.write_log(\n EdgeSeverity.EDGE_SEVERITY_NOTICE,\n errors.LogMessage.get_code(),\n 'requested to stop; disconnecting now')\n\n async def recover_from_error(self):\n # Consume all messages until sync.\n\n while True:\n\n if not self.buffer.take_message():\n await self.wait_for_message(report_idling=True)\n mtype = self.buffer.get_message_type()\n\n if mtype == b'S':\n await self.sync()\n return\n else:\n self.buffer.discard_message()\n\n cdef write_error(self, exc):\n self.write_edgedb_error(execute.interpret_simple_error(exc))\n\n cdef write_edgedb_error(self, exc):\n cdef:\n WriteBuffer buf\n int16_t fields_len\n\n if self.debug and not isinstance(exc, errors.BackendUnavailableError):\n self.debug_print('EXCEPTION', type(exc).__name__, exc)\n from edb.common.markup import dump\n dump(exc)\n\n if debug.flags.server and not isinstance(\n exc, errors.BackendUnavailableError\n ):\n self.loop.call_exception_handler({\n 'message': (\n 'an error in edgedb protocol'\n ),\n 'exception': exc,\n 'protocol': self,\n 'transport': self._transport,\n })\n\n fields = {}\n if isinstance(exc, errors.EdgeDBError):\n fields.update(exc._attrs)\n if isinstance(exc, errors.TransactionSerializationError):\n metrics.transaction_serialization_errors.inc(\n 1.0, self.get_tenant_label()\n )\n\n try:\n formatted_error = exc.__formatted_error__\n except AttributeError:\n try:\n formatted_error = ''.join(\n traceback.format_exception(\n type(exc), exc, exc.__traceback__,\n limit=50))\n except Exception:\n formatted_error = 'could not serialize error traceback'\n\n fields[base_errors.FIELD_SERVER_TRACEBACK] = formatted_error\n\n buf = WriteBuffer.new_message(b'E')\n buf.write_byte(EdgeSeverity.EDGE_SEVERITY_ERROR)\n buf.write_int32(exc.get_code())\n buf.write_len_prefixed_utf8(str(exc))\n buf.write_int16(len(fields))\n for k, v in fields.items():\n buf.write_int16(k)\n buf.write_len_prefixed_utf8(str(v))\n buf.end_message()\n\n self.write(buf)\n\n async def interpret_error(self, exc):\n dbv = self.get_dbview()\n return await execute.interpret_error(\n exc,\n dbv._db,\n global_schema_pickle=dbv.get_global_schema_pickle(),\n user_schema_pickle=dbv.get_user_schema_pickle(),\n )\n\n cdef write_status(self, bytes name, bytes value):\n cdef:\n WriteBuffer buf\n\n buf = WriteBuffer.new_message(b'S')\n buf.write_len_prefixed_bytes(name)\n buf.write_len_prefixed_bytes(value)\n buf.end_message()\n\n self.write(buf)\n\n cdef write_log(self, EdgeSeverity severity, uint32_t code, str message):\n cdef:\n WriteBuffer buf\n\n if severity >= EdgeSeverity.EDGE_SEVERITY_ERROR:\n # This is an assertion.\n raise errors.InternalServerError(\n 'emitting a log message with severity=ERROR')\n\n buf = WriteBuffer.new_message(b'L')\n buf.write_byte(severity)\n buf.write_int32(code)\n buf.write_len_prefixed_utf8(message)\n buf.write_int16(0) # number of annotations\n buf.end_message()\n\n self.write(buf)\n\n cdef sync_status(self):\n cdef:\n WriteBuffer buf\n dbview.DatabaseConnectionView _dbview\n\n buf = WriteBuffer.new_message(b'Z')\n buf.write_int16(0) # no annotations\n\n # NOTE: EdgeDB and PostgreSQL current statuses can disagree.\n # For example, Postres can be \"PQTRANS_INTRANS\" whereas EdgeDB\n # would be \"PQTRANS_INERROR\". This can happen becuase some of\n # EdgeDB errors can happen at the compile stage, not even\n # reaching Postgres.\n\n _dbview = self.get_dbview()\n if _dbview.in_tx_error():\n buf.write_byte(b'E')\n elif _dbview.in_tx():\n buf.write_byte(b'T')\n else:\n buf.write_byte(b'I')\n\n return buf.end_message()\n\n cdef fallthrough(self):\n cdef:\n char mtype = self.buffer.get_message_type()\n\n if mtype == b'H':\n # Flush\n self.buffer.discard_message()\n self.flush()\n\n elif mtype == b'X':\n # Terminate\n self.buffer.discard_message()\n self.close()\n\n else:\n raise errors.BinaryProtocolError(\n f'unexpected message type {chr(mtype)!r}')\n\n def connection_made(self, transport):\n if self._con_status != EDGECON_NEW:\n raise errors.BinaryProtocolError(\n 'invalid connection status while establishing the connection')\n super().connection_made(transport)\n\n cdef _main_task_created(self):\n self.server.on_binary_client_connected(self)\n\n def connection_lost(self, exc):\n self.server.on_binary_client_disconnected(self)\n super().connection_lost(exc)\n\n @contextlib.asynccontextmanager\n async def _with_dump_restore_pgcon(self):\n self._in_dump_restore = True\n try:\n async with self.with_pgcon() as conn:\n yield conn\n finally:\n self._in_dump_restore = False\n # If backpressure was being applied during the operation, release it.\n # `resume_reading` is idempotent.\n self._transport.resume_reading()\n\n async def dump(self):\n cdef:\n WriteBuffer msg_buf\n dbview.DatabaseConnectionView _dbview\n uint64_t flags\n\n # Parse the \"Dump\" message\n if self.protocol_version >= (3, 0):\n self.ignore_annotations()\n flags = self.buffer.read_int64()\n include_secrets = flags & messages.DumpFlag.DUMP_SECRETS\n else:\n headers = self.parse_headers()\n include_secrets = headers.get(QUERY_HEADER_DUMP_SECRETS) == b'\\x01'\n\n self.buffer.finish_message()\n\n _dbview = self.get_dbview()\n if _dbview.txid:\n raise errors.ProtocolError(\n 'DUMP must not be executed while in transaction'\n )\n\n is_superuser, _ = _dbview.get_permissions()\n if not is_superuser:\n raise errors.DisabledCapabilityError(\n f'role {_dbview._role_name} does not have permission to '\n f'perform dump'\n )\n\n server = self.server\n compiler_pool = server.get_compiler_pool()\n\n dbname = _dbview.dbname\n async with self._with_dump_restore_pgcon() as pgcon:\n # To avoid having races, we want to:\n #\n # 1. start a transaction;\n #\n # 2. in the compiler process we connect to that transaction\n # and re-introspect the schema in it.\n #\n # 3. all dump worker pg connection would work on the same\n # connection.\n #\n # This guarantees that every pg connection and the compiler work\n # with the same DB state.\n\n await pgcon.sql_execute(\n b'''START TRANSACTION\n ISOLATION LEVEL SERIALIZABLE\n READ ONLY\n DEFERRABLE;\n\n -- Disable transaction or query execution timeout\n -- limits. Both clients and the server can be slow\n -- during the dump/restore process.\n SET LOCAL idle_in_transaction_session_timeout = 0;\n SET LOCAL statement_timeout = 0;\n ''',\n )\n\n user_schema_json = await server.introspect_user_schema_json(pgcon)\n global_schema_json = (\n await server.introspect_global_schema_json(pgcon)\n )\n db_config_json = await server.introspect_db_config(pgcon)\n dump_protocol = self.max_protocol\n\n schema_ddl, schema_dynamic_ddl, schema_ids, blocks = (\n await compiler_pool.describe_database_dump(\n user_schema_json,\n global_schema_json,\n db_config_json,\n dump_protocol,\n include_secrets,\n )\n )\n\n if schema_dynamic_ddl:\n for query in schema_dynamic_ddl:\n result = await pgcon.sql_fetch_val(query.encode('utf-8'))\n if result:\n schema_ddl += '\\n' + result.decode('utf-8')\n\n msg_buf = WriteBuffer.new_message(b'@') # DumpHeader\n\n msg_buf.write_int16(4) # number of key-value pairs\n msg_buf.write_int16(DUMP_HEADER_BLOCK_TYPE)\n msg_buf.write_len_prefixed_bytes(DUMP_HEADER_BLOCK_TYPE_INFO)\n msg_buf.write_int16(DUMP_HEADER_SERVER_VER)\n msg_buf.write_len_prefixed_utf8(str(buildmeta.get_version()))\n msg_buf.write_int16(DUMP_HEADER_SERVER_CATALOG_VERSION)\n msg_buf.write_int32(8)\n msg_buf.write_int64(buildmeta.EDGEDB_CATALOG_VERSION)\n msg_buf.write_int16(DUMP_HEADER_SERVER_TIME)\n msg_buf.write_len_prefixed_utf8(str(int(time.time())))\n\n msg_buf.write_int16(dump_protocol[0])\n msg_buf.write_int16(dump_protocol[1])\n msg_buf.write_len_prefixed_utf8(schema_ddl)\n\n msg_buf.write_int32(len(schema_ids))\n for (tn, td, tid) in schema_ids:\n msg_buf.write_len_prefixed_utf8(tn)\n msg_buf.write_len_prefixed_utf8(td)\n assert len(tid) == 16\n msg_buf.write_bytes(tid) # uuid\n\n msg_buf.write_int32(len(blocks))\n for block in blocks:\n assert len(block.schema_object_id.bytes) == 16\n msg_buf.write_bytes(block.schema_object_id.bytes) # uuid\n msg_buf.write_len_prefixed_bytes(block.type_desc)\n\n msg_buf.write_int16(len(block.schema_deps))\n for depid in block.schema_deps:\n assert len(depid.bytes) == 16\n msg_buf.write_bytes(depid.bytes) # uuid\n\n self._transport.write(memoryview(msg_buf.end_message()))\n self.flush()\n\n blocks_queue = collections.deque(blocks)\n output_queue = asyncio.Queue(maxsize=2)\n\n async with asyncio.TaskGroup() as g:\n g.create_task(pgcon.dump(\n blocks_queue,\n output_queue,\n DUMP_BLOCK_SIZE,\n ))\n\n nstops = 0\n while True:\n if self._cancelled:\n raise ConnectionAbortedError\n\n out = await output_queue.get()\n if out is None:\n nstops += 1\n if nstops == 1:\n # we only have one worker right now\n break\n else:\n block, block_num, data = out\n\n msg_buf = WriteBuffer.new_message(b'=') # DumpBlock\n msg_buf.write_int16(4) # number of key-value pairs\n\n msg_buf.write_int16(DUMP_HEADER_BLOCK_TYPE)\n msg_buf.write_len_prefixed_bytes(\n DUMP_HEADER_BLOCK_TYPE_DATA)\n msg_buf.write_int16(DUMP_HEADER_BLOCK_ID)\n msg_buf.write_len_prefixed_bytes(\n block.schema_object_id.bytes)\n msg_buf.write_int16(DUMP_HEADER_BLOCK_NUM)\n msg_buf.write_len_prefixed_bytes(\n str(block_num).encode())\n msg_buf.write_int16(DUMP_HEADER_BLOCK_DATA)\n msg_buf.write_len_prefixed_buffer(data)\n\n self._transport.write(memoryview(msg_buf.end_message()))\n if self._write_waiter:\n await self._write_waiter\n\n await pgcon.sql_execute(b\"ROLLBACK;\")\n\n msg_buf = WriteBuffer.new_message(b'C') # CommandComplete\n msg_buf.write_int16(0) # no annotations\n msg_buf.write_int64(0) # capabilities\n msg_buf.write_len_prefixed_bytes(b'DUMP')\n msg_buf.write_bytes(sertypes.NULL_TYPE_ID.bytes)\n msg_buf.write_len_prefixed_bytes(b'')\n self.write(msg_buf.end_message())\n self.flush()\n\n async def _execute_utility_stmt(self, eql: str, pgcon):\n cdef dbview.DatabaseConnectionView _dbview = self.get_dbview()\n\n cfg_ser = self.server.compilation_config_serializer\n query_req = rpc.CompilationRequest(\n source=edgeql.Source.from_string(eql),\n protocol_version=self.protocol_version,\n schema_version=_dbview.schema_version,\n compilation_config_serializer=cfg_ser,\n role_name=self.username,\n branch_name=self.dbname,\n )\n\n compiled = await _dbview.parse(query_req)\n query_unit_group = compiled.query_unit_group\n assert len(query_unit_group) == 1\n query_unit = query_unit_group[0]\n\n try:\n _dbview.start(query_unit)\n await pgcon.sql_execute(query_unit.sql)\n except Exception:\n _dbview.on_error()\n if (\n query_unit.tx_commit and\n not pgcon.in_tx() and\n _dbview.in_tx()\n ):\n # The COMMIT command has failed. Our Postgres connection\n # isn't in a transaction anymore. Abort the transaction\n # in dbview.\n _dbview.abort_tx()\n raise\n else:\n _dbview.on_success(query_unit, {})\n # _execute_utility_stmt is only used in restore(), where the state\n # serializer is not coming with the COMMIT command. However, we try\n # to keep the state serializer here anyways in case of future use\n if query_unit_group.state_serializer is not None:\n _dbview.set_state_serializer(query_unit_group.state_serializer)\n\n async def restore(self):\n cdef:\n WriteBuffer msg_buf\n char mtype\n dbview.DatabaseConnectionView _dbview\n\n _dbview = self.get_dbview()\n if _dbview.txid:\n raise errors.ProtocolError(\n 'RESTORE must not be executed while in transaction'\n )\n is_superuser, _ = _dbview.get_permissions()\n if not is_superuser:\n raise errors.DisabledCapabilityError(\n f'role {_dbview._role_name} does not have permission to '\n f'perform restore'\n )\n\n if _dbview.get_state_serializer() is None:\n await _dbview.reload_state_serializer()\n\n # Parse the \"Restore\" message\n if self.buffer.read_int16() != 0: # number of attributes\n raise errors.BinaryProtocolError('unexpected attributes')\n self.buffer.read_int16() # discard -j level\n\n # Now parse the embedded \"DumpHeader\" message:\n\n server = self.server\n compiler_pool = server.get_compiler_pool()\n\n global_schema_pickle = _dbview.get_global_schema_pickle()\n user_schema_pickle = _dbview.get_user_schema_pickle()\n\n dump_server_ver_str = None\n cat_ver = None\n headers_num = self.buffer.read_int16()\n for _ in range(headers_num):\n hdrname = self.buffer.read_int16()\n hdrval = self.buffer.read_len_prefixed_bytes()\n if hdrname == DUMP_HEADER_SERVER_VER:\n dump_server_ver_str = hdrval.decode('utf-8')\n if hdrname == DUMP_HEADER_SERVER_CATALOG_VERSION:\n cat_ver = parse_catalog_version_header(hdrval)\n\n proto_major = self.buffer.read_int16()\n proto_minor = self.buffer.read_int16()\n proto = (proto_major, proto_minor)\n if proto > DUMP_VER_MAX or proto < DUMP_VER_MIN:\n raise errors.ProtocolError(\n f'unsupported dump version {proto_major}.{proto_minor}')\n\n schema_ddl = self.buffer.read_len_prefixed_bytes()\n\n ids_num = self.buffer.read_int32()\n schema_ids = []\n for _ in range(ids_num):\n schema_ids.append((\n self.buffer.read_len_prefixed_utf8(),\n self.buffer.read_len_prefixed_utf8(),\n self.buffer.read_bytes(16),\n ))\n\n block_num = self.buffer.read_int32()\n blocks = []\n for _ in range(block_num):\n blocks.append((\n self.buffer.read_bytes(16),\n self.buffer.read_len_prefixed_bytes(),\n ))\n\n # Ignore deps info\n for _ in range(self.buffer.read_int16()):\n self.buffer.read_bytes(16)\n\n self.buffer.finish_message()\n dbname = _dbview.dbname\n\n async with self._with_dump_restore_pgcon() as pgcon:\n _dbview.decode_state(sertypes.NULL_TYPE_ID.bytes, b'')\n await self._execute_utility_stmt(\n 'START TRANSACTION',\n pgcon,\n )\n\n try:\n await pgcon.sql_execute(\n b'''\n -- Drop isolation level.\n SET TRANSACTION ISOLATION LEVEL READ COMMITTED;\n -- Disable transaction or query execution timeout\n -- limits. Both clients and the server can be slow\n -- during the dump/restore process.\n SET LOCAL idle_in_transaction_session_timeout = 0;\n SET LOCAL statement_timeout = 0;\n ''',\n )\n\n schema_sql_units, restore_blocks, tables, repopulate_units = \\\n await compiler_pool.describe_database_restore(\n user_schema_pickle,\n global_schema_pickle,\n dump_server_ver_str,\n cat_ver,\n schema_ddl,\n schema_ids,\n blocks,\n proto,\n )\n\n for query_unit in schema_sql_units:\n new_types = None\n _dbview.start(query_unit)\n\n try:\n if query_unit.config_ops:\n for op in query_unit.config_ops:\n if op.scope is config.ConfigScope.INSTANCE:\n raise errors.ProtocolError(\n 'CONFIGURE INSTANCE cannot be executed'\n ' in dump restore'\n )\n\n if query_unit.sql:\n if query_unit.ddl_stmt_id:\n await pgcon.parse_execute(query=query_unit)\n ddl_ret = pgcon.load_last_ddl_return(query_unit)\n if ddl_ret and ddl_ret['new_types']:\n new_types = ddl_ret['new_types']\n else:\n await pgcon.sql_execute(query_unit.sql)\n except Exception:\n _dbview.on_error()\n raise\n else:\n _dbview.on_success(query_unit, new_types)\n\n restore_blocks = {\n b.schema_object_id: b\n for b in restore_blocks\n }\n\n disable_trigger_q = ''\n enable_trigger_q = ''\n for table in tables:\n disable_trigger_q += (\n f'ALTER TABLE {table} DISABLE TRIGGER ALL;'\n )\n enable_trigger_q += (\n f'ALTER TABLE {table} ENABLE TRIGGER ALL;'\n )\n\n await pgcon.sql_execute(disable_trigger_q.encode())\n\n # Send \"RestoreReady\" message\n msg = WriteBuffer.new_message(b'+')\n msg.write_int16(0) # no annotations\n msg.write_int16(1) # -j1\n self.write(msg.end_message())\n self.flush()\n\n while True:\n if not self.buffer.take_message():\n # Don't report idling when restoring a dump.\n # This is an edge case and the client might be\n # legitimately slow.\n await self.wait_for_message(report_idling=False)\n mtype = self.buffer.get_message_type()\n\n if mtype == b'=': # RestoreBlock\n block_type = None\n block_id = None\n block_num = None\n block_data = None\n\n num_headers = self.buffer.read_int16()\n for _ in range(num_headers):\n header = self.buffer.read_int16()\n if header == DUMP_HEADER_BLOCK_TYPE:\n block_type = self.buffer.read_len_prefixed_bytes()\n elif header == DUMP_HEADER_BLOCK_ID:\n block_id = self.buffer.read_len_prefixed_bytes()\n block_id = pg_UUID(block_id)\n elif header == DUMP_HEADER_BLOCK_NUM:\n block_num = self.buffer.read_len_prefixed_bytes()\n elif header == DUMP_HEADER_BLOCK_DATA:\n block_data = self.buffer.read_len_prefixed_bytes()\n\n self.buffer.finish_message()\n\n if (block_type is None or block_id is None\n or block_num is None or block_data is None):\n raise errors.ProtocolError('incomplete data block')\n\n restore_block = restore_blocks[block_id]\n type_id_map = self._build_type_id_map_for_restore_mending(\n restore_block)\n self._transport.pause_reading()\n await pgcon.restore(restore_block, block_data, type_id_map)\n self._transport.resume_reading()\n\n elif mtype == b'.': # RestoreEof\n self.buffer.finish_message()\n break\n\n else:\n self.fallthrough()\n\n for repopulate_unit in repopulate_units:\n await pgcon.sql_execute(repopulate_unit.encode())\n\n await pgcon.sql_execute(enable_trigger_q.encode())\n\n except Exception:\n await pgcon.sql_execute(b'ROLLBACK')\n _dbview.abort_tx()\n raise\n\n else:\n await self._execute_utility_stmt('COMMIT', pgcon)\n\n execute.signal_side_effects(_dbview, dbview.SideEffects.SchemaChanges)\n await self.tenant.introspect_db(dbname)\n\n if _dbview.is_state_desc_changed():\n self.write(self.make_state_data_description_msg())\n\n state_tid, state_data = _dbview.encode_state()\n\n msg = WriteBuffer.new_message(b'C') # CommandComplete\n msg.write_int16(0) # no annotations\n msg.write_int64(0) # capabilities\n msg.write_len_prefixed_bytes(b'RESTORE')\n msg.write_bytes(state_tid.bytes)\n msg.write_len_prefixed_bytes(state_data)\n self.write(msg.end_message())\n self.flush()\n\n def _build_type_id_map_for_restore_mending(self, restore_block):\n type_map = {}\n descriptor_stack = []\n\n if not restore_block.data_mending_desc:\n return type_map\n\n descriptor_stack.append(restore_block.data_mending_desc)\n while descriptor_stack:\n desc_tuple = descriptor_stack.pop()\n for desc in desc_tuple:\n if desc is not None:\n type_map[desc.schema_type_id] = (\n self.get_dbview().resolve_backend_type_id(\n desc.schema_type_id,\n )\n )\n\n descriptor_stack.append(desc.elements)\n\n return type_map\n\n\n@cython.final\ncdef class VirtualTransport:\n def __init__(self, transport):\n self.buf = WriteBuffer.new()\n self.closed = False\n self.transport = transport\n\n def write(self, data):\n self.buf.write_bytes(bytes(data))\n\n def _get_data(self):\n return bytes(self.buf)\n\n def is_closing(self):\n return self.closed\n\n def close(self):\n self.closed = True\n\n def abort(self):\n self.closed = True\n\n def get_extra_info(self, name, default=None):\n return self.transport.get_extra_info(name, default)\n\n\nasync def eval_buffer(\n server,\n tenant,\n database: str,\n data: bytes,\n conn_params: dict[str, str],\n protocol_version: edbdef.ProtocolVersion,\n auth_data: bytes,\n transport: srvargs.ServerConnTransport,\n tcp_transport: asyncio.Transport,\n):\n cdef:\n VirtualTransport vtr\n EdgeConnection proto\n\n vtr = VirtualTransport(tcp_transport)\n\n proto = new_edge_connection(\n server,\n tenant,\n passive=True,\n auth_data=auth_data,\n transport=transport,\n conn_params=conn_params,\n protocol_version=protocol_version,\n )\n\n proto.connection_made(vtr)\n if vtr.is_closing() or proto._main_task is None:\n raise RuntimeError(\n 'cannot process the request, the server is shutting down')\n\n # HACK: In the tunneled protocol we don't have the username when\n # we create the dbview, so put in an empty username. It will be\n # filled in once auth is called.\n proto.username = ''\n\n try:\n await proto._start_connection(database)\n proto.data_received(data)\n await proto._main_task\n except Exception as ex:\n proto.connection_lost(ex)\n else:\n proto.connection_lost(None)\n\n data = vtr._get_data()\n return data\n\n\ndef new_edge_connection(\n server,\n tenant,\n *,\n external_auth: bool = False,\n passive: bool = False,\n transport: srvargs.ServerConnTransport = (\n srvargs.ServerConnTransport.TCP),\n auth_data: bytes = b'',\n protocol_version: edbdef.ProtocolVersion = edbdef.CURRENT_PROTOCOL,\n conn_params: dict[str, str] | None = None,\n connection_made_at: float | None = None,\n):\n return EdgeConnection(\n server,\n tenant,\n external_auth=external_auth,\n passive=passive,\n transport=transport,\n auth_data=auth_data,\n protocol_version=protocol_version,\n conn_params=conn_params,\n connection_made_at=connection_made_at,\n )\n\n\nasync def run_script(\n server,\n tenant,\n database: str,\n user: str,\n script: str,\n) -> None:\n cdef:\n EdgeConnection conn\n dbview.CompiledQuery compiled\n dbview.DatabaseConnectionView _dbview\n conn = new_edge_connection(server, tenant)\n conn.username = user\n await conn._start_connection(database)\n try:\n _dbview = conn.get_dbview()\n cfg_ser = server.compilation_config_serializer\n compiled = await _dbview.parse(\n rpc.CompilationRequest(\n source=edgeql.Source.from_string(script),\n protocol_version=conn.protocol_version,\n schema_version=_dbview.schema_version,\n compilation_config_serializer=cfg_ser,\n output_format=FMT_NONE,\n role_name=user,\n branch_name=database,\n ),\n )\n compiled.tag = \"gel/startup-script\"\n if len(compiled.query_unit_group) > 1:\n await conn._execute_script(compiled, b'')\n else:\n await conn._execute(compiled, b'', use_prep_stmt=0)\n except Exception as e:\n exc = await conn.interpret_error(e)\n if isinstance(exc, errors.EdgeDBError):\n raise exc from None\n else:\n raise exc\n finally:\n conn.close()\n\n\ncdef _extract_key_vars(\n qug: dbstate.QueryUnitGroup,\n query_req: rpc.CompilationRequest,\n args: bytes\n):\n cdef:\n FRBuffer in_buf\n char *p\n int32_t recv_args\n int32_t decl_args\n ssize_t in_len\n\n frb_init(\n &in_buf,\n cpython.PyBytes_AS_STRING(args),\n cpython.Py_SIZE(args))\n\n keys = qug.graphql_key_variables\n\n in_type_args = qug.in_type_args or ()\n decl_args = len(in_type_args)\n if args:\n recv_args = hton.unpack_int32(frb_read(&in_buf, 4))\n else:\n recv_args = 0\n if recv_args != decl_args:\n raise errors.InputDataError(\n f\"invalid argument count, \"\n f\"expected: {decl_args}, got: {recv_args}\")\n\n vals = {}\n\n for param in in_type_args:\n frb_read(&in_buf, 4) # reserved\n needed = param.name in keys\n\n in_len = hton.unpack_int32(frb_read(&in_buf, 4))\n if not needed:\n if in_len > 0:\n frb_read(&in_buf, in_len)\n continue\n\n if in_len < 0:\n val = None\n else:\n p = frb_read(&in_buf, in_len)\n\n # Very hacky and minimal decoding support.\n if param.typename == 'std::str':\n val = cpython.PyUnicode_DecodeUTF8(p, in_len, NULL)\n elif param.typename == 'std::bool':\n val = p[0] != 0\n else:\n raise AssertionError(\n f'unsupported type for graphql introspection: '\n f'{param.typename}'\n )\n\n vals[param.name] = val\n\n # Extracted arguments come from the NormalizedSource.\n query_vars = query_req.source.variables()\n for name in keys:\n if name.startswith('__edb_arg_'):\n vals[name] = query_vars[name]\n\n return vals\n" }, { "path": "edb/server/protocol/consts.pxi", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2019-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nDEF DUMP_BLOCK_SIZE = 1024 * 1024 * 10\n\nDEF DUMP_HEADER_BLOCK_TYPE = 101\nDEF DUMP_HEADER_BLOCK_TYPE_INFO = b'I'\nDEF DUMP_HEADER_BLOCK_TYPE_DATA = b'D'\n\nDEF DUMP_HEADER_SERVER_TIME = 102\nDEF DUMP_HEADER_SERVER_VER = 103\nDEF DUMP_HEADER_BLOCKS_INFO = 104\nDEF DUMP_HEADER_SERVER_CATALOG_VERSION = 105\n\nDEF DUMP_HEADER_BLOCK_ID = 110\nDEF DUMP_HEADER_BLOCK_NUM = 111\nDEF DUMP_HEADER_BLOCK_DATA = 112\n" }, { "path": "edb/server/protocol/cpythonx.pxd", "content": "# Copyright (C) 2016-present the asyncpg authors and contributors\n# \n#\n# This module is part of asyncpg and is released under\n# the Apache 2.0 License: http://www.apache.org/licenses/LICENSE-2.0\n\n\ncdef extern from \"Python.h\":\n object PyLong_FromUnicodeObject(\n object u, int base)\n" }, { "path": "edb/server/protocol/edgeql_ext.pyx", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2019-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nimport decimal\nimport http\nimport json\nimport urllib.parse\n\nimport immutables\n\nfrom edb import errors\nfrom edb import edgeql\nfrom edb.server import defines as edbdef\nfrom edb.server.protocol import execute\n\nfrom edb.schema import schema as s_schema\n\nfrom edb.common import debug\nfrom edb.common import markup\n\nfrom edb.edgeql import qltypes\n\nfrom edb.server import compiler\nfrom edb.server import config\nfrom edb.server.compiler import enums\nfrom edb.server.dbview cimport dbview\nfrom edb.server.pgproto.pgproto cimport WriteBuffer\n\n\nasync def handle_request(\n object request,\n object response,\n dbview.Database db,\n str role_name,\n list args,\n object tenant,\n):\n if args != []:\n response.body = b'Unknown path'\n response.status = http.HTTPStatus.NOT_FOUND\n response.close_connection = True\n return\n\n variables = None\n globals_ = None\n query = None\n config = None\n\n try:\n if request.method == b'POST':\n if request.content_type and b'json' in request.content_type:\n body = json.loads(request.body, parse_float=decimal.Decimal)\n if not isinstance(body, dict):\n raise TypeError(\n 'the body of the request must be a JSON object')\n query = body.get('query')\n variables = body.get('variables')\n globals_ = body.get('globals')\n config = body.get('config')\n else:\n raise TypeError(\n 'unable to interpret EdgeQL POST request')\n\n elif request.method == b'GET':\n if request.url.query:\n url_query = request.url.query.decode('ascii')\n qs = urllib.parse.parse_qs(url_query)\n\n query = qs.get('query')\n if query is not None:\n query = query[0]\n\n variables = qs.get('variables')\n if variables is not None:\n try:\n variables = json.loads(variables[0])\n except Exception:\n raise TypeError(\n '\"variables\" must be a JSON object')\n\n globals_ = qs.get('globals')\n if globals_ is not None:\n try:\n globals_ = json.loads(globals_[0])\n except Exception:\n raise TypeError(\n '\"globals\" must be a JSON object')\n config = qs.get('config')\n if config is not None:\n try:\n config = json.loads(config[0])\n except Exception:\n raise TypeError(\n '\"config\" must be a JSON object')\n\n else:\n raise TypeError('expected a GET or a POST request')\n\n if not query:\n raise TypeError('invalid EdgeQL request: query is missing')\n\n if variables is not None and not isinstance(variables, dict):\n raise TypeError('\"variables\" must be a JSON object')\n\n if globals_ is not None and not isinstance(globals_, dict):\n raise TypeError('\"globals\" must be a JSON object')\n\n if config is not None and not isinstance(config, dict):\n raise TypeError('\"config\" must be a JSON object')\n\n except Exception as ex:\n if debug.flags.server:\n markup.dump(ex)\n\n response.body = str(ex).encode()\n response.status = http.HTTPStatus.BAD_REQUEST\n response.close_connection = True\n return\n\n response.status = http.HTTPStatus.OK\n response.content_type = b'application/json'\n try:\n result = await execute.parse_execute_json(\n db,\n query,\n role_name=role_name,\n variables=variables or {},\n globals_=globals_,\n session_config=config,\n )\n except Exception as ex:\n if debug.flags.server:\n markup.dump(ex)\n\n ex = await execute.interpret_error(ex, db)\n\n response.body = json.dumps({'error': ex.to_json()}).encode()\n else:\n response.body = b'{\"data\":' + result + b'}'\n" }, { "path": "edb/server/protocol/execute.pxd", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2024-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\nfrom edb.server.pgproto.pgproto cimport WriteBuffer\n\n\ncdef class ExecutionGroup:\n cdef:\n object group\n list bind_datas\n\n cdef append(self, object query_unit, WriteBuffer bind_data=?)\n" }, { "path": "edb/server/protocol/execute.pyi", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2019-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\nfrom typing import (\n Any,\n Mapping,\n Optional,\n)\nimport immutables\n\nfrom edb import errors\nfrom edb.server import compiler\nfrom edb.server import defines as edbdef\nfrom edb.server.compiler import sertypes\nfrom edb.server.dbview import dbview\n\nasync def describe(\n db: dbview.Database,\n query: str,\n *,\n query_cache_enabled: Optional[bool] = None,\n allow_capabilities: compiler.Capability = (\n compiler.Capability.MODIFICATIONS),\n query_tag: str | None = None,\n role_name: str,\n) -> sertypes.TypeDesc:\n ...\n\nasync def parse_execute_json(\n db: dbview.Database,\n query: str,\n *,\n variables: Mapping[str, Any] = immutables.Map(),\n globals_: Optional[Mapping[str, Any]] = None,\n output_format: compiler.OutputFormat = compiler.OutputFormat.JSON,\n query_cache_enabled: Optional[bool] = None,\n cached_globally: bool = False,\n use_metrics: bool = True,\n tx_isolation: edbdef.TxIsolationLevel | None = None,\n query_tag: str | None = None,\n role_name: str | None = None,\n) -> bytes:\n ...\n\nasync def interpret_error(\n exc: Exception,\n db: dbview.Database,\n *,\n global_schema_pickle: object=None,\n user_schema_pickle: object=None,\n from_graphql: bool=False,\n) -> errors.EdgeDBError:\n ...\n" }, { "path": "edb/server/protocol/execute.pyx", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2019-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\nfrom typing import (\n Any,\n Mapping,\n Optional,\n)\n\nfrom edgedb import scram\n\nimport asyncio\nimport base64\nimport decimal\nimport hashlib\nimport json\nimport logging\n\nimport immutables\n\nfrom edb import errors\nfrom edb.common import debug\n\nfrom edb import edgeql\nfrom edb.edgeql import qltypes\n\nfrom edb.pgsql.parser import exceptions as parser_errors\n\nfrom edb.server import compiler\nfrom edb.server import config\nfrom edb.server import defines as edbdef\nfrom edb.server import metrics\nfrom edb.server.compiler import dbstate\nfrom edb.server.compiler import errormech\nfrom edb.server.compiler cimport rpc\nfrom edb.server.compiler import sertypes\nfrom edb.server.dbview cimport dbview\nfrom edb.server.protocol cimport args_ser\nfrom edb.server.protocol cimport frontend\nfrom edb.server.protocol import ai_ext\nfrom edb.server.pgcon cimport pgcon\nfrom edb.server.pgcon import errors as pgerror\n\n\ncdef object logger = logging.getLogger('edb.server')\n\ncdef object FMT_NONE = compiler.OutputFormat.NONE\ncdef WriteBuffer NO_ARGS = args_ser.combine_raw_args()\n\n\ncdef class ExecutionGroup:\n def __cinit__(self):\n self.group = compiler.QueryUnitGroup()\n self.bind_datas = []\n\n cdef append(self, object query_unit, WriteBuffer bind_data=NO_ARGS):\n self.group.append(query_unit, serialize=False)\n self.bind_datas.append(bind_data)\n\n async def execute(\n self,\n pgcon.PGConnection be_conn,\n object dbv, # can be DatabaseConnectionView or Database\n fe_conn: frontend.AbstractFrontendConnection = None,\n bytes state = None,\n bint needs_commit_state = False,\n ):\n cdef int dbver\n\n rv = None\n async with be_conn.parse_execute_script_context():\n dbver = dbv.dbver\n parse_array = [False] * len(self.group)\n be_conn.send_query_unit_group(\n self.group,\n True, # sync\n self.bind_datas,\n state,\n 0, # start\n len(self.group), # end\n dbver,\n parse_array,\n None, # query_prefix\n needs_commit_state,\n )\n if state is not None:\n await be_conn.wait_for_state_resp(\n state,\n state_sync=needs_commit_state,\n needs_commit_state=needs_commit_state,\n )\n for i, unit in enumerate(self.group):\n ignore_data = unit.output_format == FMT_NONE\n rv = await be_conn.wait_for_command(\n unit,\n parse_array[i],\n dbver,\n ignore_data=ignore_data,\n fe_conn=None if ignore_data else fe_conn,\n )\n return rv\n\n\ncpdef ExecutionGroup build_cache_persistence_units(\n pairs: list[tuple[rpc.CompilationRequest, compiler.QueryUnitGroup]],\n ExecutionGroup group = None,\n):\n if group is None:\n group = ExecutionGroup()\n insert_sql = b'''\n INSERT INTO \"edgedb\".\"_query_cache\"\n (\"key\", \"schema_version\", \"input\", \"output\", \"evict\")\n VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (key) DO NOTHING\n '''\n sql_hash = hashlib.sha1(insert_sql).hexdigest().encode('latin1')\n for request, units in pairs:\n # FIXME: this is temporary; drop this assertion when we support scripts\n assert len(units) == 1\n query_unit = units[0]\n\n assert query_unit.cache_sql is not None\n persist, evict = query_unit.cache_sql\n\n serialized_result = units.maybe_get_serialized(0)\n assert serialized_result is not None\n\n if evict:\n group.append(compiler.QueryUnit(sql=evict, status=b''))\n if persist:\n group.append(compiler.QueryUnit(sql=persist, status=b''))\n group.append(\n compiler.QueryUnit(sql=insert_sql, sql_hash=sql_hash, status=b''),\n args_ser.combine_raw_args((\n query_unit.cache_key.bytes,\n query_unit.user_schema_version.bytes,\n request.serialize(),\n serialized_result,\n evict,\n )),\n )\n return group\n\n\nasync def describe(\n db: dbview.Database,\n query: str,\n *,\n query_cache_enabled: Optional[bool] = None,\n allow_capabilities: compiler.Capability = compiler.Capability.MODIFICATIONS,\n query_tag: str | None = None,\n role_name: str,\n) -> sertypes.TypeDesc:\n dbv = await _get_transient_dbv(db, role_name=role_name)\n _, compiled = await _parse(\n dbv,\n query,\n query_cache_enabled=query_cache_enabled,\n allow_capabilities=allow_capabilities,\n )\n if query_tag:\n compiled.tag = query_tag\n\n try:\n desc = sertypes.parse(\n compiled.query_unit_group.out_type_data,\n edbdef.CURRENT_PROTOCOL,\n )\n finally:\n db.tenant.remove_dbview(dbv)\n\n return desc\n\n\nasync def _get_transient_dbv(\n db: dbview.Database,\n *,\n query_cache_enabled: Optional[bool] = None,\n role_name: str,\n) -> dbview.DatabaseConnectionView:\n if query_cache_enabled is None:\n query_cache_enabled = not (\n debug.flags.disable_qcache or debug.flags.edgeql_compile)\n\n tenant = db.tenant\n dbv = await tenant.new_dbview(\n dbname=db.name,\n query_cache=query_cache_enabled,\n protocol_version=edbdef.CURRENT_PROTOCOL,\n role_name=role_name,\n )\n dbv.is_transient = True\n\n return dbv\n\n\nasync def _parse(\n dbv: dbview.DatabaseConnectionView,\n query: str,\n *,\n input_format: compiler.InputFormat = compiler.InputFormat.BINARY,\n output_format: compiler.OutputFormat = compiler.OutputFormat.BINARY,\n allow_capabilities: compiler.Capability = compiler.Capability.MODIFICATIONS,\n use_metrics: bool = True,\n cached_globally: bool = False,\n query_cache_enabled: Optional[bool] = None,\n) -> tuple[\n rpc.CompilationRequest,\n dbview.CompiledQuery,\n]:\n db = dbv._db\n tenant = db.tenant\n\n if use_metrics:\n metrics.query_size.observe(\n len(query.encode('utf-8')), tenant.get_instance_name(), 'edgeql'\n )\n\n query_req = rpc.CompilationRequest(\n source=edgeql.Source.from_string(query),\n protocol_version=edbdef.CURRENT_PROTOCOL,\n schema_version=dbv.schema_version,\n compilation_config_serializer=db.server.compilation_config_serializer,\n input_format=input_format,\n output_format=output_format,\n session_config=dbv.get_session_config(),\n database_config=dbv.get_database_config(),\n system_config=dbv.get_compilation_system_config(),\n role_name=dbv._role_name,\n )\n\n compiled = await dbv.parse(\n query_req,\n cached_globally=cached_globally,\n use_metrics=use_metrics,\n allow_capabilities=allow_capabilities,\n )\n\n return query_req, compiled\n\n\n# TODO: can we merge execute and execute_script?\nasync def execute(\n be_conn: pgcon.PGConnection,\n dbv: dbview.DatabaseConnectionView,\n compiled: dbview.CompiledQuery,\n bind_args: bytes,\n *,\n fe_conn: frontend.AbstractFrontendConnection = None,\n use_prep_stmt: bint = False,\n tx_isolation: edbdef.TxIsolationLevel | None = None,\n query_req: Optional[rpc.CompilationRequest] = None,\n):\n cdef:\n bytes state = None, orig_state = None\n WriteBuffer bound_args_buf\n bint needs_commit_state = False\n\n query_unit = compiled.query_unit_group[0]\n\n if not dbv.in_tx():\n orig_state = state = dbv.serialize_state()\n needs_commit_state = dbv.needs_commit_after_state_sync()\n\n new_types = None\n server = dbv.server\n tenant = dbv.tenant\n\n data = None\n\n try:\n if be_conn.last_state == state:\n # the current status in be_conn is in sync with dbview, skip the\n # state restoring\n state = None\n dbv.start(query_unit)\n if query_unit.create_db_template:\n await tenant.on_before_create_db_from_template(\n query_unit.create_db_template,\n dbv.dbname,\n query_unit.create_db_mode,\n )\n if query_unit.drop_db:\n await tenant.on_before_drop_db(\n query_unit.drop_db,\n dbv.dbname,\n close_frontend_conns=query_unit.drop_db_reset_connections,\n )\n\n if query_unit.early_non_tx_sql:\n # Sync state non transactionally\n await be_conn.sql_fetch(b'select 1', state=state)\n for sql in query_unit.early_non_tx_sql:\n await be_conn.sql_execute(sql)\n\n if query_unit.system_config:\n # execute_system_config() always sync state in a separate tx,\n # so we don't need to pass down the needs_commit_state here\n await execute_system_config(be_conn, dbv, query_unit, state)\n else:\n config_ops = query_unit.config_ops\n\n if query_unit.sql:\n if query_unit.user_schema:\n await be_conn.parse_execute(\n query=query_unit,\n state=state,\n needs_commit_state=needs_commit_state,\n )\n if query_unit.ddl_stmt_id is not None:\n ddl_ret = be_conn.load_last_ddl_return(query_unit)\n if ddl_ret and ddl_ret['new_types']:\n new_types = ddl_ret['new_types']\n else:\n converted_args: Optional[list[args_ser.ConvertedArg]] = None\n if query_unit.server_param_conversions:\n converted_args = (await _convert_parameters(\n dbv,\n compiled,\n query_unit.server_param_conversions,\n bind_args,\n )).get(0, None)\n\n data_types = []\n bound_args_buf = args_ser.recode_bind_args(\n dbv,\n compiled,\n bind_args,\n converted_args,\n None,\n data_types,\n )\n\n assert not (query_unit.database_config\n and query_unit.needs_readback), (\n \"needs_readback+database_config must use execute_script\"\n )\n read_data = (\n query_unit.needs_readback or query_unit.is_explain)\n\n data = await be_conn.parse_execute(\n query=query_unit,\n fe_conn=fe_conn if not read_data else None,\n bind_data=bound_args_buf,\n param_data_types=data_types,\n use_prep_stmt=use_prep_stmt,\n state=state,\n needs_commit_state=needs_commit_state,\n dbver=dbv.dbver,\n use_pending_func_cache=compiled.use_pending_func_cache,\n tx_isolation=tx_isolation,\n query_prefix=compiled.make_query_prefix(),\n )\n\n if query_unit.needs_readback and data:\n config_ops = [\n config.Operation.from_json(r[0][1:])\n for r in data\n ]\n\n if query_unit.is_explain:\n # Go back to the compiler pool to analyze\n # the explain output.\n compiler_pool = server.get_compiler_pool()\n r = await compiler_pool.analyze_explain_output(\n query_unit.query_asts, data\n )\n buf = WriteBuffer.new_message(b'D')\n buf.write_int16(1) # 1 column\n buf.write_len_prefixed_bytes(r)\n fe_conn.write(buf.end_message())\n\n if state is not None:\n # state is restored, clear orig_state so that we can\n # set be_conn.last_state correctly later\n orig_state = None\n\n if query_unit.tx_savepoint_rollback:\n dbv.rollback_tx_to_savepoint(query_unit.sp_name)\n\n if query_unit.tx_savepoint_declare:\n dbv.declare_savepoint(\n query_unit.sp_name, query_unit.sp_id)\n\n if query_unit.create_db_template:\n try:\n await tenant.on_after_create_db_from_template(\n query_unit.create_db,\n query_unit.create_db_template,\n query_unit.create_db_mode,\n )\n except Exception:\n # Clean up the database if we failed to restore into it.\n # TODO: Is it worth having 'ready' flag that we set after\n # the database is fully set up, and use that to clean up\n # databases where a crash prevented doing this cleanup?\n db_name = f'{tenant.tenant_id}_{query_unit.create_db}'\n await be_conn.sql_execute(\n b'drop database \"%s\"' % db_name.encode('utf-8')\n )\n raise\n\n if query_unit.create_db:\n await tenant.introspect_db(query_unit.create_db)\n\n if query_unit.drop_db:\n tenant.on_after_drop_db(query_unit.drop_db)\n\n if config_ops:\n await dbv.apply_config_ops(be_conn, config_ops)\n\n if query_unit.user_schema and debug.flags.delta_validate_reflection:\n global_schema = (\n query_unit.global_schema or dbv.get_global_schema_pickle())\n new_user_schema = await dbv.tenant._debug_introspect(\n be_conn, global_schema)\n compiler_pool = dbv.server.get_compiler_pool()\n await compiler_pool.validate_schema_equivalence(\n query_unit.user_schema,\n new_user_schema,\n global_schema,\n dbv._last_comp_state,\n )\n query_unit.user_schema = new_user_schema\n\n except Exception as ex:\n if isinstance(ex, pgerror.BackendError):\n # If we made schema changes, include the new schema in the\n # exception so that it can be used when interpreting.\n if query_unit.user_schema:\n ex._user_schema = query_unit.user_schema\n\n # If we get an undefined function error, this is probably\n # because of a pgfunc cache invalidation race condition,\n # where another frontend dropped the function but we\n # haven't processed the message yet. We are going to\n # trigger a client retry (via errormech), but we also want\n # to invalidate the cache entry, in cache we haven't\n # processed the message by the retry.\n if (\n query_req\n and ex.code_is(pgerror.ERROR_UNDEFINED_FUNCTION)\n ):\n dbv._db.invalidate_cache_entry_object(query_req)\n\n if query_unit.source_map:\n ex._from_sql = True\n\n dbv.on_error()\n\n if query_unit.tx_commit and not be_conn.in_tx() and dbv.in_tx():\n # The COMMIT command has failed. Our Postgres connection\n # isn't in a transaction anymore. Abort the transaction\n # in dbview.\n dbv.abort_tx()\n raise\n else:\n side_effects = dbv.on_success(query_unit, new_types)\n state_serializer = compiled.query_unit_group.state_serializer\n if state_serializer is not None:\n dbv.set_state_serializer(state_serializer)\n if side_effects:\n await process_side_effects(dbv, side_effects, be_conn)\n if not dbv.in_tx() and not query_unit.tx_rollback and query_unit.sql:\n state = dbv.serialize_state()\n if state is not orig_state:\n # In 3 cases the state is changed:\n # 1. The non-tx query changed the state\n # 2. The state is synced with dbview (orig_state is None)\n # 3. We came out from a transaction (orig_state is None)\n # Excluding two special case when the state is NOT changed:\n # 1. An orphan ROLLBACK command without a paring start tx\n # 2. There was no SQL, so the state can't have been synced.\n be_conn.last_state = state\n be_conn.state_reset_needs_commit = (\n dbv.needs_commit_after_state_sync())\n if compiled.recompiled_cache:\n for req, qu_group in compiled.recompiled_cache:\n dbv.cache_compiled_query(req, qu_group)\n finally:\n if query_unit.drop_db:\n tenant.allow_database_connections(query_unit.drop_db)\n\n return data\n\n\nasync def _convert_parameters(\n dbv: dbview.DatabaseConnectionView,\n compiled: dbview.CompiledQuery,\n server_param_conversions: list[dbstate.ServerParamConversion],\n bind_args: bytes,\n) -> dict[int, list[args_ser.ConvertedArg]]:\n \"\"\"\n If there are server param conversions, compute them now so that they are\n injected into the recoded bind args later.\n \"\"\"\n\n param_conversions: list[args_ser.ParamConversion] = (\n args_ser.get_param_conversions(\n dbv,\n server_param_conversions,\n bind_args,\n compiled.extra_blobs,\n )\n )\n\n # Cache converted args which may be used in multiple units\n converted_args_cache: list[Optional[args_ser.ConvertedArg]] = (\n [None] * len(param_conversions)\n )\n\n unit_group = compiled.query_unit_group\n\n # First check for conversions which should be done in batches\n ai_text_embedding_conversion_indexes: list[int] = []\n ai_text_embedding_conversions: list[args_ser.ParamConversion] = []\n for unit_index, converted_params_indexes in (\n unit_group.unit_converted_param_indexes.items()\n ):\n for conversion_index in converted_params_indexes:\n conversion = param_conversions[conversion_index]\n\n conversion_name: str = conversion.get_conversion_name()\n\n if conversion_name == 'ai_text_embedding':\n ai_text_embedding_conversion_indexes.append(conversion_index)\n ai_text_embedding_conversions.append(conversion)\n\n # Compute batched conversions and store them in cache\n if ai_text_embedding_conversions:\n converted_args = (\n await _batch_convert_ai_text_embedding(\n dbv, ai_text_embedding_conversions\n )\n )\n for conversion_index, converted_arg in zip(\n ai_text_embedding_conversion_indexes,\n converted_args,\n ):\n converted_args_cache[conversion_index] = converted_arg\n\n # Do the remaining conversions\n converted_args: dict[int, list[args_ser.ConvertedArg]] = {}\n for unit_index, converted_params_indexes in (\n unit_group.unit_converted_param_indexes.items()\n ):\n unit_converted_args: list[args_ser.ParamConversion] = []\n\n for conversion_index in converted_params_indexes:\n # Check for a cached conversion arg\n if converted_arg := converted_args_cache[conversion_index]:\n unit_converted_args.append(converted_arg)\n continue\n\n # Do the conversion\n converted_arg = await _convert_parameter(\n param_conversions[conversion_index]\n )\n\n unit_converted_args.append(converted_arg)\n converted_args_cache[conversion_index] = converted_arg\n\n if unit_converted_args:\n converted_args[unit_index] = unit_converted_args\n\n return converted_args\n\n\nasync def _convert_parameter(\n conversion: args_ser.ParamConversion,\n) -> args_ser.ConvertedArg:\n\n conversion_name = conversion.get_conversion_name()\n\n # We receive the encoded param data from the bind_args or extra blobs\n # and decode it manually.\n if (\n conversion_name == 'cast_int64_to_str'\n or conversion_name == 'cast_int64_to_str_volatile'\n ):\n decoded_param_data = conversion.param_as_int()\n return args_ser.ConvertedArgStr.new(\n str(decoded_param_data)\n )\n\n elif conversion_name == 'cast_int64_to_float64':\n decoded_param_data = conversion.param_as_int()\n\n return args_ser.ConvertedArgFloat64.new(\n float(decoded_param_data)\n )\n\n elif conversion_name == 'join_str_array':\n decoded_param_data = conversion.param_as_array_of_str()\n\n separator = conversion.get_additional_info()[0]\n return args_ser.ConvertedArgStr.new(\n separator.join(decoded_param_data)\n )\n\n elif conversion_name == 'ai_text_embedding':\n raise RuntimeError(f'conversion should be batched: {conversion_name}')\n\n else:\n raise errors.QueryError(\n f'unknown param conversion: {conversion_name}'\n )\n\n\nasync def _batch_convert_ai_text_embedding(\n dbv: dbview.DatabaseConnectionView,\n conversions: list[args_ser.ParamConversion],\n) -> list[args_ser.ConvertedArg]:\n embeddings_inputs: list[tuple[str, str]] = [\n (\n conversion_data.get_additional_info()[0],\n conversion_data.param_as_str(),\n )\n for conversion_data in conversions\n ]\n\n tenant = dbv.tenant\n db = tenant.maybe_get_db(dbname=dbv.dbname)\n assert db is not None\n\n embeddings_result = await ai_ext.generate_embeddings_for_texts(\n db,\n tenant.get_http_client(originator=\"ai/index\"),\n embeddings_inputs,\n )\n if embeddings_result.too_long:\n long_input = embeddings_inputs[embeddings_result.too_long[0]][1][:100]\n raise errors.QueryError(\n f'Search text exceeds maximum input token length: {long_input}...'\n )\n if not embeddings_result.success:\n raise RuntimeError('failed to get embeddings')\n\n return [\n args_ser.ConvertedArgListFloat32.new(\n embeddings\n )\n for embeddings in embeddings_result.success\n ]\n\n\nasync def execute_script(\n conn: pgcon.PGConnection,\n dbv: dbview.DatabaseConnectionView,\n compiled: dbview.CompiledQuery,\n bind_args: bytes,\n *,\n query_req: Optional[rpc.CompilationRequest] = None,\n fe_conn: Optional[frontend.AbstractFrontendConnection],\n):\n cdef:\n bytes state = None, orig_state = None\n ssize_t sent = 0\n bint in_tx, sync, no_sync\n object user_schema, extensions, ext_config_settings, cached_reflection\n object global_schema, roles\n WriteBuffer bind_data\n int dbver = dbv.dbver\n bint parse, needs_commit_state = False\n\n user_schema = extensions = ext_config_settings = cached_reflection = None\n feature_used_metrics = None\n global_schema = roles = None\n unit_group = compiled.query_unit_group\n query_prefix = compiled.make_query_prefix()\n query_unit = None\n\n sync = False\n no_sync = False\n in_tx = dbv.in_tx()\n if not in_tx:\n orig_state = state = dbv.serialize_state()\n needs_commit_state = dbv.needs_commit_after_state_sync()\n\n data = None\n\n try:\n if conn.last_state == state:\n # the current status in be_conn is in sync with dbview, skip the\n # state restoring\n state = None\n async with conn.parse_execute_script_context():\n\n converted_args: Optional[dict[int, list[args_ser.ConvertedArg]]] = None\n if unit_group.server_param_conversions:\n converted_args = await _convert_parameters(\n dbv,\n compiled,\n unit_group.server_param_conversions,\n bind_args,\n )\n\n parse_array = [False] * len(unit_group)\n for idx, query_unit in enumerate(unit_group):\n if fe_conn is not None and fe_conn.cancelled:\n raise ConnectionAbortedError\n\n assert not query_unit.is_explain\n\n # XXX: pull out?\n # We want to minimize the round trips we need to make, so\n # ideally we buffer up everything, send it once, and then issue\n # one SYNC. This gets messed up if there are commands where\n # we need to read back information, though, such as SET GLOBAL.\n #\n # Because of that, we look for the next command that\n # needs read back (probably there won't be one!), and\n # execute everything up to that point at once,\n # finished by a FLUSH.\n if idx >= sent:\n no_sync = False\n for n in range(idx, len(unit_group)):\n ng = unit_group[n]\n if ng.ddl_stmt_id or ng.needs_readback:\n sent = n + 1\n if ng.needs_readback:\n no_sync = True\n break\n else:\n sent = len(unit_group)\n\n sync = sent == len(unit_group) and not no_sync\n\n bind_array = args_ser.recode_bind_args_for_script(\n dbv,\n compiled,\n bind_args,\n converted_args,\n idx,\n sent,\n )\n\n dbver = dbv.dbver\n conn.send_query_unit_group(\n unit_group,\n sync,\n bind_array,\n state,\n idx,\n sent,\n dbver,\n parse_array,\n query_prefix,\n needs_commit_state,\n )\n\n if idx == 0 and state is not None:\n await conn.wait_for_state_resp(\n state,\n state_sync=needs_commit_state,\n needs_commit_state=needs_commit_state,\n )\n conn.state_reset_needs_commit = needs_commit_state\n # state is restored, clear orig_state so that we can\n # set conn.last_state correctly later\n orig_state = None\n\n new_types = None\n dbv.start_implicit(query_unit)\n config_ops = query_unit.config_ops\n\n if query_unit.user_schema:\n user_schema = query_unit.user_schema\n extensions = query_unit.extensions\n ext_config_settings = query_unit.ext_config_settings\n cached_reflection = query_unit.cached_reflection\n feature_used_metrics = query_unit.feature_used_metrics\n\n if query_unit.global_schema:\n global_schema = query_unit.global_schema\n roles = query_unit.roles\n\n if query_unit.sql:\n parse = parse_array[idx]\n fe_output = query_unit.output_format != FMT_NONE\n ignore_data = (\n not fe_output\n and not query_unit.needs_readback\n )\n data = await conn.wait_for_command(\n query_unit,\n parse,\n dbver,\n ignore_data=ignore_data,\n fe_conn=fe_conn if fe_output else None,\n )\n\n if query_unit.ddl_stmt_id:\n ddl_ret = conn.load_last_ddl_return(query_unit)\n if ddl_ret and ddl_ret['new_types']:\n new_types = ddl_ret['new_types']\n\n if query_unit.needs_readback and data:\n config_ops = [\n config.Operation.from_json(r[0][1:])\n for r in data\n ]\n\n if config_ops:\n await dbv.apply_config_ops(conn, config_ops)\n\n side_effects = dbv.on_success(query_unit, new_types)\n if side_effects:\n raise errors.InternalServerError(\n \"Side-effects in implicit transaction!\"\n )\n\n # Need to sync before calling process_side_effects, which will\n # look at the database. Also, want to sync before we record success,\n # since sync could fail.\n if sent and not sync:\n sync = True\n await conn.sync()\n\n except Exception as e:\n dbv.on_error()\n\n if isinstance(e, pgerror.BackendError):\n # Include the new schema in the exception so that it can be\n # used when interpreting.\n e._user_schema = dbv.get_user_schema_pickle()\n\n # If we get an undefined function error, this is probably\n # because of a pgfunc cache invalidation race condition,\n # where another frontend dropped the function but we\n # haven't processed the message yet. We are going to\n # trigger a client retry (via errormech), but we also want\n # to invalidate the cache entry, in cache we haven't\n # processed the message by the retry.\n if (\n query_req\n and e.code_is(pgerror.ERROR_UNDEFINED_FUNCTION)\n ):\n dbv._db.invalidate_cache_entry_object(query_req)\n\n if query_unit and query_unit.source_map:\n e._from_sql = True\n\n if not in_tx and dbv.in_tx():\n # Abort the implicit transaction\n dbv.abort_tx()\n\n # If something went wrong that is *not* on the backend side, force\n # an error to occur on the SQL side.\n if not isinstance(e, pgerror.BackendError):\n await conn.force_error()\n\n raise\n\n else:\n updated_user_schema = False\n if user_schema and debug.flags.delta_validate_reflection:\n cur_global_schema = (\n global_schema or dbv.get_global_schema_pickle())\n new_user_schema = await dbv.tenant._debug_introspect(\n conn, cur_global_schema)\n compiler_pool = dbv.server.get_compiler_pool()\n await compiler_pool.validate_schema_equivalence(\n user_schema,\n new_user_schema,\n cur_global_schema,\n dbv._last_comp_state,\n )\n user_schema = new_user_schema\n updated_user_schema = True\n\n if not in_tx:\n side_effects = dbv.commit_implicit_tx(\n user_schema,\n extensions,\n ext_config_settings,\n global_schema,\n roles,\n cached_reflection,\n feature_used_metrics,\n )\n if side_effects:\n await process_side_effects(dbv, side_effects, conn)\n state = dbv.serialize_state()\n if state is not orig_state:\n conn.last_state = state\n conn.state_reset_needs_commit = (\n dbv.needs_commit_after_state_sync())\n elif updated_user_schema:\n dbv._in_tx_user_schema_pickle = user_schema\n\n if unit_group.state_serializer is not None:\n dbv.set_state_serializer(unit_group.state_serializer)\n\n finally:\n if sent and not sync:\n await conn.sync()\n\n return data\n\n\nasync def execute_system_config(\n conn: pgcon.PGConnection,\n dbv: dbview.DatabaseConnectionView,\n query_unit: compiler.QueryUnit,\n state: bytes | None,\n):\n if query_unit.is_system_config:\n dbv.server.before_alter_system_config()\n\n # Sync state\n await conn.sql_fetch(b'select 1', state=state)\n\n if query_unit.sql:\n data = await conn.sql_fetch_col(query_unit.sql)\n else:\n data = None\n\n if data:\n # Prefer encoded op produced by the SQL command.\n if data[0][0] != 0x01:\n raise errors.InternalServerError(\n f\"unexpected JSONB version produced by SQL statement for \"\n f\"CONFIGURE INSTANCE: {data[0][0]}\"\n )\n config_ops = [config.Operation.from_json(r[1:]) for r in data]\n else:\n # Otherwise, fall back to staticly evaluated op.\n config_ops = query_unit.config_ops\n await dbv.apply_config_ops(conn, config_ops)\n\n await conn.sql_execute(b'delete from _config_cache')\n\n # If this is a backend configuration setting we also\n # need to make sure it has been loaded.\n if query_unit.backend_config:\n await conn.sql_execute(b'SELECT pg_reload_conf()')\n\n\nasync def process_side_effects(dbv, side_effects, conn):\n signal_side_effects(dbv, side_effects)\n\n if side_effects & dbview.SideEffects.DatabaseConfigChanges:\n tenant = dbv.tenant\n await tenant.process_local_database_config_change(conn, dbv.dbname)\n\n\ndef signal_side_effects(dbv, side_effects):\n tenant = dbv.tenant\n if not tenant.accept_new_tasks:\n return\n\n if side_effects & dbview.SideEffects.SchemaChanges:\n tenant.create_task(\n tenant.signal_sysevent(\n 'schema-changes',\n dbname=dbv.dbname,\n ),\n interruptable=False,\n )\n\n if side_effects & dbview.SideEffects.GlobalSchemaChanges:\n tenant.create_task(\n tenant.signal_sysevent(\n 'global-schema-changes',\n ),\n interruptable=False,\n )\n\n if side_effects & dbview.SideEffects.DatabaseConfigChanges:\n tenant.create_task(\n tenant.signal_sysevent(\n 'database-config-changes',\n dbname=dbv.dbname,\n ),\n interruptable=False,\n )\n\n if side_effects & dbview.SideEffects.DatabaseChanges:\n tenant.create_task(\n tenant.signal_sysevent(\n 'database-changes',\n ),\n interruptable=False,\n )\n\n if side_effects & dbview.SideEffects.InstanceConfigChanges:\n tenant.create_task(\n tenant.signal_sysevent(\n 'system-config-changes',\n ),\n interruptable=False,\n )\n\n\nasync def parse_execute_json(\n db: dbview.Database,\n query: str,\n *,\n variables: Mapping[str, Any] = immutables.Map(),\n globals_: Optional[Mapping[str, Any]] = None,\n session_config: Optional[Mapping[str, Any]] = None,\n output_format: compiler.OutputFormat = compiler.OutputFormat.JSON,\n query_cache_enabled: Optional[bool] = None,\n # WARNING: only set cached_globally to True when the query is\n # strictly referring to only shared stable objects in user schema\n # or anything from std schema, for example:\n # YES: select ext::auth::UIConfig { ... }\n # NO: select default::User { ... }\n cached_globally: bool = False,\n use_metrics: bool = True,\n tx_isolation: edbdef.TxIsolationLevel | None = None,\n query_tag: str | None = None,\n role_name: str | None = None,\n) -> bytes:\n if role_name is None:\n role_name = edbdef.EDGEDB_SUPERUSER\n\n dbv: dbview.DatabaseConnectionView = await _get_transient_dbv(\n db,\n query_cache_enabled=query_cache_enabled,\n role_name=role_name,\n )\n dbv.decode_json_session_config(session_config)\n query_req, compiled = await _parse(\n dbv,\n query,\n input_format=compiler.InputFormat.JSON,\n output_format=output_format,\n allow_capabilities=compiler.Capability.MODIFICATIONS,\n use_metrics=use_metrics,\n cached_globally=cached_globally,\n )\n if query_tag:\n compiled.tag = query_tag\n\n tenant = db.tenant\n async with tenant.with_pgcon(db.name) as pgcon:\n try:\n return await execute_json(\n pgcon,\n dbv,\n compiled,\n variables=variables,\n globals_=globals_,\n tx_isolation=tx_isolation,\n query_req=query_req,\n )\n finally:\n tenant.remove_dbview(dbv)\n\n\nasync def execute_json(\n be_conn: pgcon.PGConnection,\n dbv: dbview.DatabaseConnectionView,\n compiled: dbview.CompiledQuery,\n variables: Mapping[str, Any] = immutables.Map(),\n globals_: Optional[Mapping[str, Any]] = None,\n *,\n fe_conn: Optional[frontend.AbstractFrontendConnection] = None,\n use_prep_stmt: bint = False,\n tx_isolation: edbdef.TxIsolationLevel | None = None,\n query_req: Optional[rpc.CompilationRequest] = None,\n) -> bytes:\n if globals_ is None:\n globals_ = {}\n\n if compiled.query_unit_group.json_permissions:\n # Inject any required permissions into the globals json.\n\n superuser, available_permissions = dbv.get_permissions()\n\n for permission in compiled.query_unit_group.json_permissions:\n if permission in globals_:\n raise RuntimeError(\n f\"Permission cannot be passed as globals: '{permission}'\"\n )\n\n globals_[permission] = (\n superuser or permission in available_permissions\n )\n\n # TODO: only when needed? in a less dodgy way??\n for k, v in dbv._sys_globals.items():\n if k in globals_:\n raise RuntimeError(\n f\"System global '{k}' cannot be explicitly specified\"\n )\n globals_[k] = v\n\n dbv.set_globals(immutables.Map({\n \"__::__edb_json_globals__\": config.SettingValue(\n name=\"__::__edb_json_globals__\",\n value=_encode_json_value(globals_),\n source='global',\n scope=qltypes.ConfigScope.GLOBAL,\n )\n }))\n\n qug = compiled.query_unit_group\n\n args = []\n if qug.in_type_args:\n for param in qug.in_type_args:\n value = variables.get(param.name)\n args.append(value)\n\n bind_args = _encode_args(args)\n\n force_script = any(x.needs_readback for x in qug)\n if len(qug) > 1 or force_script:\n if tx_isolation is not None:\n raise errors.InternalServerError(\n \"execute_script does not support \"\n \"modified transaction isolation\"\n )\n data = await execute_script(\n be_conn,\n dbv,\n compiled,\n bind_args,\n fe_conn=fe_conn,\n query_req=query_req,\n )\n else:\n if tx_isolation is not None:\n if dbv.in_tx():\n raise errors.InternalServerError(\n \"cannot run statement with alternate transaction \"\n \"isolation: already in a transaction\"\n )\n\n query_unit = compiled.query_unit_group[0]\n if not query_unit.is_transactional:\n raise errors.InternalServerError(\n \"cannot run statement with alternate transaction \"\n \"isolation: statement is not transactional\"\n )\n\n data = await execute(\n be_conn,\n dbv,\n compiled,\n bind_args,\n fe_conn=fe_conn,\n tx_isolation=tx_isolation,\n query_req=query_req,\n )\n\n if fe_conn is None:\n if not data or len(data) > 1 or len(data[0]) != 1:\n raise errors.InternalServerError(\n f'received incorrect response data for a JSON query')\n\n return data[0][0]\n else:\n return None\n\n\nclass DecimalEncoder(json.JSONEncoder):\n def encode(self, obj):\n if isinstance(obj, dict):\n return '{' + ', '.join(\n f'{self.encode(k)}: {self.encode(v)}'\n for (k, v) in obj.items()\n ) + '}'\n if isinstance(obj, list):\n return '[' + ', '.join(map(self.encode, obj)) + ']'\n if isinstance(obj, bytes):\n return self.encode(base64.b64encode(obj).decode())\n if isinstance(obj, decimal.Decimal):\n return f'{obj:f}'\n return super().encode(obj)\n\n\ncdef bytes _encode_json_value(object val):\n jarg = json.dumps(val, cls=DecimalEncoder)\n\n return b'\\x01' + jarg.encode('utf-8')\n\n\ncdef bytes _encode_args(list args):\n cdef:\n WriteBuffer out_buf = WriteBuffer.new()\n\n if args:\n out_buf.write_int32(len(args))\n for arg in args:\n out_buf.write_int32(0) # reserved\n if arg is None:\n out_buf.write_int32(-1)\n else:\n jval = _encode_json_value(arg)\n out_buf.write_int32(len(jval))\n out_buf.write_bytes(jval)\n\n return bytes(out_buf)\n\n\ncdef _check_for_ise(exc):\n # Unwrap ExceptionGroup that has only one Exception\n if isinstance(exc, BaseExceptionGroup) and len(exc.exceptions) == 1:\n exc = exc.exceptions[0]\n\n if not isinstance(exc, errors.EdgeDBError):\n # TODO(rename): change URL once we can\n nexc = errors.InternalServerError(\n f'{type(exc).__name__}: {exc}',\n hint=(\n f'This is most likely a bug in Gel. '\n f'Please consider opening an issue ticket '\n f'at https://github.com/edgedb/edgedb/issues/new'\n f'?template=bug_report.md'\n ),\n ).with_traceback(exc.__traceback__)\n formatted = getattr(exc, '__formatted_error__', None)\n if formatted:\n nexc.__formatted_error__ = formatted\n if isinstance(exc, BaseExceptionGroup):\n nexc.__cause__ = exc.with_traceback(None)\n exc = nexc\n\n return exc\n\n\nasync def interpret_error(\n exc: Exception,\n db: dbview.Database,\n *,\n global_schema_pickle: object=None,\n user_schema_pickle: object=None,\n from_graphql: bool=False,\n) -> Exception:\n\n if isinstance(exc, RecursionError):\n exc = errors.UnsupportedFeatureError(\n \"The query caused the compiler \"\n \"stack to overflow. It is likely too deeply nested.\",\n hint=(\n \"If the query does not contain deep nesting, \"\n \"this may be a bug.\"\n ),\n )\n\n elif isinstance(exc, pgerror.BackendError):\n try:\n from_sql = getattr(exc, '_from_sql', False)\n source_map = getattr(exc, '_source_map', None)\n fields = exc.fields\n\n static_exc = errormech.static_interpret_backend_error(\n fields, from_graphql=from_graphql\n )\n\n # only use the backend if schema is required\n if static_exc is errormech.SchemaRequired:\n # Grab the schema from the exception first, if it is present.\n user_schema_pickle = (\n getattr(exc, '_user_schema', None)\n or user_schema_pickle\n or db.user_schema_pickle\n )\n global_schema_pickle = (\n global_schema_pickle or db._index._global_schema_pickle\n )\n compiler_pool = db._index._server.get_compiler_pool()\n exc = await compiler_pool.interpret_backend_error(\n user_schema_pickle,\n global_schema_pickle,\n fields,\n from_graphql,\n )\n\n elif isinstance(static_exc, (\n errors.DuplicateDatabaseDefinitionError,\n errors.UnknownDatabaseError)):\n tenant_id = db.tenant.tenant_id\n message = static_exc.args[0].replace(f'{tenant_id}_', '')\n exc = type(static_exc)(message)\n else:\n exc = static_exc\n\n if from_sql and isinstance(exc, errors.InternalServerError):\n exc = errors.ExecutionError(*exc.args)\n\n # Translate error position for SQL queries if we can\n if source_map and isinstance(exc, errors.EdgeDBError):\n if 'P' in fields:\n errors.EdgeDBError.set_position(\n exc,\n source_map.translate(int(fields['P'])),\n None,\n )\n\n # Include hint/detail from SQL queries also, if we haven't\n # produced our own.\n if from_sql and isinstance(exc, errors.EdgeDBError):\n if 'H' in fields or 'D' in fields:\n hint = exc.hint or fields.get('H')\n details = exc.details or fields.get('D')\n # ... there is some sort of cython bug/\"feature\"\n # involving the type annotation above which causes\n # exc.set_hint_and_details to fail, so we copy it\n # to a new variable.\n exc2: object = exc\n exc2.set_hint_and_details(hint, details)\n\n except Exception as e:\n from edb.common import debug\n if debug.flags.server:\n debug.dump(e)\n\n exc = RuntimeError(\n 'unhandled error while calling interpret_backend_error(); '\n 'run with EDGEDB_DEBUG_SERVER to debug.')\n\n elif isinstance(exc, parser_errors.PSqlParseError):\n exc = errormech.static_interpret_psql_parse_error(exc)\n\n return _check_for_ise(exc)\n\n\ndef interpret_simple_error(\n exc: Exception,\n) -> Exception:\n \"\"\"Intepret a protocol error not associated with a query or schema\"\"\"\n\n if isinstance(exc, pgerror.BackendError):\n static_exc = errormech.static_interpret_backend_error(exc.fields)\n if static_exc is not errormech.SchemaRequired:\n exc = static_exc\n\n return _check_for_ise(exc)\n" }, { "path": "edb/server/protocol/frontend.pxd", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2016-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nfrom edb.server.dbview cimport dbview\nfrom edb.server.pgcon cimport pgcon\nfrom edb.server.pgproto.pgproto cimport ReadBuffer, WriteBuffer\n\n\ncdef class AbstractFrontendConnection:\n\n cdef write(self, WriteBuffer buf)\n cdef flush(self)\n\n\ncdef class FrontendConnection(AbstractFrontendConnection):\n\n cdef:\n str _id\n object server\n readonly object tenant\n object loop\n readonly str dbname\n str username\n dbview.Database database\n\n pgcon.PGConnection _pinned_pgcon\n bint _pinned_pgcon_in_tx\n int _get_pgcon_cc\n\n object _transport\n WriteBuffer _write_buf\n object _write_waiter\n object connection_made_at\n int _query_count\n\n ReadBuffer buffer\n object _msg_take_waiter\n\n object started_idling_at\n bint idling\n\n bint _passive_mode\n\n bint authed\n object _main_task\n bint _cancelled\n bint _stop_requested\n bint _pgcon_released_in_connection_lost\n\n bint debug\n\n object _transport_proto\n bint _external_auth\n\n cdef _after_idling(self)\n cdef _main_task_created(self)\n cdef _main_task_stopped_normally(self)\n cdef write_error(self, exc)\n cdef stop_connection(self)\n cdef abort_pinned_pgcon(self)\n cdef is_in_tx(self)\n\n cdef WriteBuffer _make_authentication_sasl_initial(self, list methods)\n cdef _expect_sasl_initial_response(self)\n cdef WriteBuffer _make_authentication_sasl_msg(\n self, bytes data, bint final)\n cdef bytes _expect_sasl_response(self)\n" }, { "path": "edb/server/protocol/frontend.pyx", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2016-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nimport asyncio\nimport contextlib\nimport logging\nimport time\n\nfrom edgedb import scram\n\nfrom edb import errors\nfrom edb.common import debug\nfrom edb.server import defines\nfrom edb.server import args as srvargs, metrics\nfrom edb.server.pgcon import errors as pgerror\n\nfrom . cimport auth_helpers\n\n\nDEF FLUSH_BUFFER_AFTER = 100_000\ncdef object logger = logging.getLogger('edb.server')\n\n\ncdef class AbstractFrontendConnection:\n\n cdef write(self, WriteBuffer buf):\n raise NotImplementedError\n\n cdef flush(self):\n raise NotImplementedError\n\n\ncdef class FrontendConnection(AbstractFrontendConnection):\n interface = \"frontend\"\n\n def __init__(\n self,\n server,\n tenant,\n *,\n passive: bool,\n transport: srvargs.ServerConnTransport,\n external_auth: bool,\n connection_made_at: float | None = None,\n ):\n self._id = server.on_binary_client_created()\n self.server = server\n self.tenant = tenant\n self.loop = server.get_loop()\n self.dbname = None\n\n self._pinned_pgcon = None\n self._pinned_pgcon_in_tx = False\n self._get_pgcon_cc = 0\n\n self.connection_made_at = connection_made_at\n self._query_count = 0\n self._transport = None\n self._write_buf = None\n self._write_waiter = None\n\n self.buffer = ReadBuffer()\n self._msg_take_waiter = None\n\n self.idling = False\n self.started_idling_at = 0.0\n\n # In \"passive\" mode the protocol is instantiated to parse and execute\n # just what's in the buffer. It cannot \"wait for message\". This\n # is used to implement binary protocol over http+fetch.\n self._passive_mode = passive\n\n self.authed = False\n self._main_task = None\n self._cancelled = False\n self._stop_requested = False\n self._pgcon_released_in_connection_lost = False\n\n self.debug = debug.flags.server_proto\n\n self._transport_proto = transport\n self._external_auth = external_auth\n\n def get_id(self):\n return self._id\n\n cdef is_in_tx(self):\n return False\n\n # backend connection\n\n def __del__(self):\n # Should not ever happen, there's a strong ref to\n # every client connection until it hits connection_lost().\n if self._pinned_pgcon is not None:\n # XXX/TODO: add test diagnostics for this and\n # fail all tests if this ever happens.\n self.abort_pinned_pgcon()\n\n async def get_pgcon(self) -> pgcon.PGConnection:\n if self._cancelled or self._pgcon_released_in_connection_lost:\n raise RuntimeError(\n 'cannot acquire a pgconn; the connection is closed')\n self._get_pgcon_cc += 1\n try:\n if self._get_pgcon_cc > 1:\n raise RuntimeError('nested get_pgcon() calls are prohibited')\n if self.is_in_tx():\n # In transaction. We must have a working pinned connection.\n if not self._pinned_pgcon_in_tx or self._pinned_pgcon is None:\n raise RuntimeError(\n 'get_pgcon(): in dbview transaction, '\n 'but `_pinned_pgcon` is None')\n return self._pinned_pgcon\n if self._pinned_pgcon is not None:\n raise RuntimeError('there is already a pinned pgcon')\n conn = await self.tenant.acquire_pgcon(self.dbname)\n self._pinned_pgcon = conn\n conn.pinned_by = self\n return conn\n except Exception:\n self._get_pgcon_cc -= 1\n raise\n\n def maybe_release_pgcon(self, pgcon.PGConnection conn):\n self._get_pgcon_cc -= 1\n if self._get_pgcon_cc < 0:\n raise RuntimeError(\n 'maybe_release_pgcon() called more times than get_pgcon()')\n if self._pinned_pgcon is not conn:\n raise RuntimeError('mismatched released connection')\n\n if self.is_in_tx():\n if self._cancelled:\n # There could be a situation where we cancel the protocol while\n # it's in a transaction. In which case we want to immediately\n # return the connection to the pool (where it would be\n # discarded and re-opened.)\n conn.pinned_by = None\n self._pinned_pgcon = None\n if not self._pgcon_released_in_connection_lost:\n self.tenant.release_pgcon(\n self.dbname,\n conn,\n discard=debug.flags.server_clobber_pg_conns,\n )\n else:\n self._pinned_pgcon_in_tx = True\n else:\n conn.pinned_by = None\n self._pinned_pgcon_in_tx = False\n self._pinned_pgcon = None\n if not self._pgcon_released_in_connection_lost:\n self.tenant.release_pgcon(\n self.dbname,\n conn,\n discard=debug.flags.server_clobber_pg_conns,\n )\n\n @contextlib.asynccontextmanager\n async def with_pgcon(self):\n con = await self.get_pgcon()\n try:\n yield con\n finally:\n self.maybe_release_pgcon(con)\n\n def on_aborted_pgcon(self, pgcon.PGConnection conn):\n try:\n self._pinned_pgcon = None\n\n if not self._pgcon_released_in_connection_lost:\n self.tenant.release_pgcon(self.dbname, conn, discard=True)\n\n if conn.aborted_with_error is not None:\n self.write_error(conn.aborted_with_error)\n finally:\n self.close() # will flush\n\n cdef abort_pinned_pgcon(self):\n if self._pinned_pgcon is not None:\n self._pinned_pgcon.pinned_by = None\n self._pinned_pgcon.abort()\n self.tenant.release_pgcon(\n self.dbname, self._pinned_pgcon, discard=True)\n self._pinned_pgcon = None\n\n # I/O write methods, implements AbstractFrontendConnection\n\n cdef write(self, WriteBuffer buf):\n # One rule for this method: don't write partial messages.\n if self._write_buf is not None:\n self._write_buf.write_buffer(buf)\n if self._write_buf.len() >= FLUSH_BUFFER_AFTER:\n self.flush()\n else:\n self._write_buf = buf\n\n cdef flush(self):\n if self._transport is None:\n # could be if the connection is lost and a coroutine\n # method is finalizing.\n raise ConnectionAbortedError\n if self._write_buf is not None and self._write_buf.len():\n buf = self._write_buf\n self._write_buf = None\n self._transport.write(memoryview(buf))\n\n def pause_writing(self):\n if self._write_waiter and not self._write_waiter.done():\n return\n self._write_waiter = self.loop.create_future()\n\n def resume_writing(self):\n if not self._write_waiter or self._write_waiter.done():\n return\n self._write_waiter.set_result(True)\n\n # I/O read methods\n\n def data_received(self, data):\n self.buffer.feed_data(data)\n if self._msg_take_waiter is not None and self.buffer.take_message():\n self._msg_take_waiter.set_result(True)\n self._msg_take_waiter = None\n\n def eof_received(self):\n pass\n\n cdef _after_idling(self):\n # Hook for EdgeConnection\n pass\n\n async def wait_for_message(self, *, bint report_idling):\n if self.buffer.take_message():\n return\n if self._passive_mode:\n raise RuntimeError('cannot wait for more messages in passive mode')\n if self._transport is None:\n # could be if the connection is lost and a coroutine\n # method is finalizing.\n raise ConnectionAbortedError\n\n self._msg_take_waiter = self.loop.create_future()\n if report_idling:\n self.idling = True\n self.started_idling_at = time.monotonic()\n\n try:\n await self._msg_take_waiter\n finally:\n self.idling = False\n\n self._after_idling()\n\n def is_idle(self, expiry_time: float):\n # A connection is idle if it awaits for the next message for\n # client for too long (even if it is in an open transaction!)\n return self.idling and self.started_idling_at < expiry_time\n\n # establishing a new connection\n\n cdef _main_task_created(self):\n pass\n\n cdef _main_task_stopped_normally(self):\n pass\n\n def get_tenant_label(self):\n if self.tenant is None:\n return \"unknown\"\n else:\n return self.tenant.get_instance_name()\n\n def connection_made(self, transport):\n if self.tenant is None:\n self._transport = transport\n self._main_task = self.loop.create_task(self.handshake())\n self._main_task_created()\n elif self.tenant.is_accepting_connections():\n self._transport = transport\n self._main_task = self.tenant.create_task(\n self.main(), interruptable=False\n )\n self._main_task_created()\n else:\n transport.abort()\n\n async def handshake(self):\n try:\n await self._handshake()\n except Exception as ex:\n if self._transport is not None:\n # If there's no transport it means that the connection\n # was aborted, in which case we don't really care about\n # reporting the exception.\n self.write_error(ex)\n self.close()\n\n if not isinstance(ex, (errors.ProtocolError,\n errors.AuthenticationError)):\n self.loop.call_exception_handler({\n 'message': (\n f'unhandled error in {self.__class__.__name__} while '\n 'accepting new connection'\n ),\n 'exception': ex,\n 'protocol': self,\n 'transport': self._transport,\n 'task': self._main_task,\n })\n\n async def _handshake(self):\n if self.tenant is None:\n self.tenant = self.server.get_default_tenant()\n if self.tenant.is_accepting_connections():\n self._main_task = self.tenant.create_task(\n self.main(), interruptable=False\n )\n else:\n if self._transport is not None:\n self._transport.abort()\n\n # main skeleton\n\n async def main_step(self, char mtype):\n raise NotImplementedError\n\n cdef write_error(self, exc):\n raise NotImplementedError\n\n async def main(self):\n cdef char mtype\n\n try:\n await self.authenticate()\n except Exception as ex:\n if self._transport is not None:\n # If there's no transport it means that the connection\n # was aborted, in which case we don't really care about\n # reporting the exception.\n self.write_error(ex)\n self.close()\n\n if not isinstance(ex, (errors.ProtocolError,\n errors.AuthenticationError)):\n self.loop.call_exception_handler({\n 'message': (\n f'unhandled error in {self.__class__.__name__} while '\n 'accepting new connection'\n ),\n 'exception': ex,\n 'protocol': self,\n 'transport': self._transport,\n 'task': self._main_task,\n })\n\n return\n\n self.authed = True\n\n try:\n while True:\n if self._cancelled:\n self.abort()\n return\n\n if self._stop_requested:\n break\n\n if not self.buffer.take_message():\n if self._passive_mode:\n # In \"passive\" mode we only parse what's in the buffer\n # and return. If there's any unparsed (incomplete) data\n # in the buffer it's an error.\n if self.buffer._length:\n raise RuntimeError(\n 'unparsed data in the read buffer')\n # Flush whatever data is in the internal buffer before\n # returning.\n self.flush()\n return\n await self.wait_for_message(report_idling=True)\n\n mtype = self.buffer.get_message_type()\n if await self.main_step(mtype):\n break\n\n except asyncio.CancelledError:\n # Happens when the connection is aborted, the backend is\n # being closed and propagates CancelledError to all\n # EdgeCon methods that await on, say, the compiler process.\n # We shouldn't have CancelledErrors otherwise, therefore,\n # in this situation we just silently exit.\n pass\n\n except ConnectionError:\n metrics.connection_errors.inc(\n 1.0, self.get_tenant_label(),\n )\n\n except pgerror.BackendQueryCancelledError:\n pass\n\n except Exception as ex:\n # We can only be here if an exception occurred during\n # handling another exception, in which case, the only\n # sane option is to abort the connection.\n\n self.loop.call_exception_handler({\n 'message': (\n 'unhandled error in edgedb protocol while '\n 'handling an error'\n ),\n 'exception': ex,\n 'protocol': self,\n 'transport': self._transport,\n 'task': self._main_task,\n })\n\n finally:\n if self._stop_requested:\n self._main_task_stopped_normally()\n self.close()\n else:\n # Abort the connection.\n # It might have already been cleaned up, but abort() is\n # safe to be called on a closed connection.\n self.abort()\n\n # shutting down the connection\n\n cdef stop_connection(self):\n pass\n\n def close(self):\n self.abort_pinned_pgcon()\n self.stop_connection()\n if self._transport is not None:\n self.flush()\n self._transport.close()\n self._transport = None\n\n def abort(self):\n self.abort_pinned_pgcon()\n self.stop_connection()\n if self._transport is not None:\n self._transport.abort()\n self._transport = None\n\n def request_stop(self):\n # Actively stop a frontend connection - this is used by the server\n # when it's stopping.\n self._stop_requested = True\n if self._msg_take_waiter is not None:\n if not self._msg_take_waiter.done():\n self._msg_take_waiter.cancel()\n\n @property\n def cancelled(self) -> bool:\n return self._cancelled\n\n def is_alive(self):\n return self._transport is not None and not self._cancelled\n\n def connection_lost(self, exc):\n # Let's talk about cancellation.\n #\n # 1. Since we need to synchronize the state between Postgres and\n # EdgeDB, we need to make sure we never do straight asyncio\n # cancellation while some operation in pgcon is in flight.\n #\n # Doing that can lead to the following few bad scenarios:\n #\n # * pgcon connction being wrecked by asyncio.CancelledError;\n #\n # * pgcon completing its operation and then, a rogue\n # CancelledError preventing us to apply the new state\n # to dbview/server config/etc.\n #\n # 2. It is safe to cancel `_msg_take_waiter` though. Cancelling it\n # would abort protocol parsing, but there's no global state that\n # needs syncing in protocol messages.\n #\n # 3. We can interrupt some operations like auth with a CancelledError.\n # Again, those operations don't mutate global state.\n\n if self.connection_made_at is not None:\n tenant_label = self.get_tenant_label()\n metrics.client_connection_duration.observe(\n time.monotonic() - self.connection_made_at,\n tenant_label,\n self.interface,\n )\n if self.authed:\n metrics.queries_per_connection.observe(\n self._query_count, tenant_label, self.interface\n )\n if isinstance(exc, ConnectionError):\n metrics.connection_errors.inc(1.0, tenant_label)\n\n if (self._msg_take_waiter is not None and\n not self._msg_take_waiter.done()):\n # We're parsing the protocol. We can abort that.\n self._msg_take_waiter.cancel()\n\n if (\n self._main_task is not None\n and not self._main_task.done()\n and not self._cancelled\n ):\n\n # The main connection handling task is up and running.\n\n # First, let's set a flag to signal that we should cancel soon;\n # after all the client has already disconnected.\n self._cancelled = True\n\n # Make sure nothing is blocked on flow control.\n # (Currently only dump uses this.)\n self.resume_writing()\n\n if not self.authed:\n # We must be still authenticating. We can abort that.\n self._main_task.cancel()\n else:\n if (\n self._pinned_pgcon is not None\n and not self._pinned_pgcon.idle\n ):\n # Looks like we have a Postgres connection acquired and\n # it's actively running some command for us. To make\n # sure we're not leaving behind a heavy query, perform\n # an explicit Postgres cancellation because a mere\n # connection drop wouldn't necessarily abort the query\n # right away). Additionally, we must discard the connection\n # as we cannot be completely sure about its state. Postgres\n # cancellation is signal-based and is addressed to a whole\n # connection and not a concrete operation. The result is\n # that we might be racing with the currently running query\n # and if that completes before the cancellation signal\n # reaches the backend, we'll be setting a trap for the\n # _next_ query that is unlucky enough to pick up this\n # Postgres backend from the connection pool.\n # TODO(fantix): hold server shutdown to complete this task\n if self.tenant.accept_new_tasks:\n self.tenant.create_task(\n self.tenant.cancel_and_discard_pgcon(\n self._pinned_pgcon, self.dbname\n ),\n interruptable=False,\n )\n # Prevent the main task from releasing the same connection\n # twice. This flag is for now only used in this case.\n self._pgcon_released_in_connection_lost = True\n\n # In all other cases, we can just wait until the `main()`\n # coroutine notices that `self._cancelled` was set.\n # It would be a mistake to cancel the main task here, as it\n # could be unpacking results from pgcon and applying them\n # to the global state.\n #\n # Ultimately, the main() coroutine will be aborted, eventually,\n # and will call `self.abort()` to shut all things down.\n else:\n # The `main()` coroutine isn't running, it means that the\n # connection is already pretty much dead. Nonetheless, call\n # abort() to make sure we've cleaned everything up properly.\n self.abort()\n\n # Authentication\n\n async def authenticate(self):\n raise NotImplementedError\n\n def _auth_jwt(self, user, database, params):\n raise NotImplementedError\n\n def _auth_trust(self, user):\n roles = self.tenant.get_roles()\n if user not in roles:\n raise errors.AuthenticationError('authentication failed')\n\n async def _authenticate(self, user, database, params):\n # The user has already been authenticated by other means\n # (such as the ability to write to a protected socket).\n if self._external_auth:\n authmethods = [\n self.server.config_settings.get_type_by_name('cfg::Trust')()\n ]\n else:\n authmethods = await self.tenant.get_auth_methods(\n user, self._transport_proto)\n\n auth_errors = {}\n\n for authmethod in authmethods:\n authmethod_name = authmethod._tspec.name.split('::')[1]\n\n try:\n if authmethod_name == 'SCRAM':\n await self._auth_scram(user)\n elif authmethod_name == 'JWT':\n self._auth_jwt(user, database, params)\n elif authmethod_name == 'Trust':\n self._auth_trust(user)\n elif authmethod_name == 'Password':\n raise errors.AuthenticationError(\n 'authentication failed: '\n 'Simple password authentication required but it is '\n 'only supported for HTTP endpoints'\n )\n elif authmethod_name == 'mTLS':\n auth_helpers.auth_mtls_with_user(self._transport, user)\n else:\n raise errors.InternalServerError(\n f'unimplemented auth method: {authmethod_name}')\n except errors.AuthenticationError as e:\n auth_errors[authmethod_name] = e\n else:\n break\n\n if len(auth_errors) == len(authmethods):\n if len(auth_errors) > 1:\n desc = \"; \".join(\n f\"{k}: {e.args[0]}\" for k, e in auth_errors.items())\n raise errors.AuthenticationError(\n f\"all authentication methods failed: {desc}\")\n else:\n raise next(iter(auth_errors.values()))\n\n role = self.tenant.get_roles().get(user)\n if not role:\n raise errors.AuthenticationError('authentication failed')\n branches = role['branches']\n if (\n '*' not in branches\n and database not in branches\n and database != defines.EDGEDB_SYSTEM_DB\n ):\n raise errors.AuthenticationError(\n f\"authentication failed: user does not have permission for \"\n f\"database branch '{database}'\"\n )\n\n cdef WriteBuffer _make_authentication_sasl_initial(self, list methods):\n raise NotImplementedError\n\n cdef _expect_sasl_initial_response(self):\n raise NotImplementedError\n\n cdef WriteBuffer _make_authentication_sasl_msg(\n self, bytes data, bint final\n ):\n raise NotImplementedError\n\n cdef bytes _expect_sasl_response(self):\n raise NotImplementedError\n\n async def _auth_scram(self, user):\n cdef WriteBuffer msg_buf\n\n # Tell the client that we require SASL SCRAM auth.\n msg_buf = self._make_authentication_sasl_initial([b'SCRAM-SHA-256'])\n self.write(msg_buf)\n self.flush()\n\n selected_mech = None\n verifier = None\n mock_auth = False\n client_nonce = None\n cb_flag = None\n done = False\n\n while not done:\n if not self.buffer.take_message():\n await self.wait_for_message(report_idling=True)\n mtype = self.buffer.get_message_type()\n\n if selected_mech is None:\n # Initial response.\n (\n selected_mech, client_first\n ) = self._expect_sasl_initial_response()\n if selected_mech != b'SCRAM-SHA-256':\n raise errors.BinaryProtocolError(\n f'client selected an invalid SASL authentication '\n f'mechanism')\n verifier, mock_auth = auth_helpers.scram_get_verifier(\n self.tenant, user)\n\n try:\n bare_offset, cb_flag, authzid, username, client_nonce = (\n scram.parse_client_first_message(client_first))\n except ValueError as e:\n raise errors.BinaryProtocolError(str(e))\n\n client_first_bare = client_first[bare_offset:]\n\n if isinstance(cb_flag, str):\n raise errors.BinaryProtocolError(\n 'malformed SCRAM message',\n details='The client selected SCRAM-SHA-256 without '\n 'channel binding, but the SCRAM message '\n 'includes channel binding data.')\n\n if authzid:\n raise errors.UnsupportedFeatureError(\n 'client uses SASL authorization identity, '\n 'which is not supported')\n\n server_nonce = scram.generate_nonce()\n server_first = scram.build_server_first_message(\n server_nonce, client_nonce,\n verifier.salt, verifier.iterations).encode('utf-8')\n\n msg_buf = self._make_authentication_sasl_msg(server_first, 0)\n self.write(msg_buf)\n self.flush()\n\n else:\n # client final message\n client_final = self._expect_sasl_response()\n try:\n cb_data, client_proof, proof_len = (\n scram.parse_client_final_message(\n client_final, client_nonce, server_nonce))\n except ValueError as e:\n raise errors.BinaryProtocolError(str(e)) from None\n\n client_final_without_proof = client_final[:-proof_len]\n\n cb_data_ok = (\n (cb_flag is False and cb_data == b'biws')\n or (cb_flag is True and cb_data == b'eSws')\n )\n if not cb_data_ok:\n raise errors.BinaryProtocolError(\n 'malformed SCRAM message',\n details='Unexpected SCRAM channel-binding attribute '\n 'in client-final-message.')\n\n if not scram.verify_client_proof(\n client_first_bare, server_first,\n client_final_without_proof,\n verifier.stored_key, client_proof):\n raise errors.AuthenticationError(\n 'authentication failed')\n\n if mock_auth:\n # This user actually does not exist, so fail here.\n raise errors.AuthenticationError(\n 'authentication failed')\n\n server_final = scram.build_server_final_message(\n client_first_bare,\n server_first,\n client_final_without_proof,\n verifier.server_key,\n ).encode('utf-8')\n\n # AuthenticationSASLFinal\n msg_buf = self._make_authentication_sasl_msg(server_final, 1)\n self.write(msg_buf)\n self.flush()\n\n done = True\n" }, { "path": "edb/server/protocol/metrics.py", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2021-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nfrom __future__ import annotations\nfrom typing import TYPE_CHECKING\nimport http\n\nfrom edb import errors\nfrom edb.server import metrics\nfrom edb.server import server\n\nfrom edb.common import debug\nfrom edb.common import markup\n\nif TYPE_CHECKING:\n from edb.server import tenant as edbtenant\n from edb.server.protocol import protocol\n\n\nasync def handle_request(\n request: protocol.HttpRequest,\n response: protocol.HttpResponse,\n tenant: edbtenant.Tenant,\n) -> None:\n try:\n if tenant is None or isinstance(tenant.server, server.Server):\n output = metrics.registry.generate()\n else:\n output = metrics.registry.generate(\n tenant=tenant.get_instance_name()\n )\n response.status = http.HTTPStatus.OK\n response.content_type = b'text/plain; version=0.0.4; charset=utf-8'\n response.body = output.encode()\n response.close_connection = True\n\n except Exception as ex:\n if debug.flags.server:\n markup.dump(ex)\n\n # XXX Fix this when LSP \"location\" objects are implemented\n ex_type = errors.InternalServerError\n\n _response_error(\n response, http.HTTPStatus.INTERNAL_SERVER_ERROR, str(ex), ex_type\n )\n\n\ndef _response_error(\n response: protocol.HttpResponse,\n status: http.HTTPStatus,\n message: str,\n ex_type: type[errors.EdgeDBError],\n) -> None:\n response.body = (\n f'Unexpected error in /metrics.\\n\\n'\n f'{ex_type.__name__}: {message}'\n ).encode()\n response.status = status\n response.close_connection = True\n" }, { "path": "edb/server/protocol/notebook_ext.pxd", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2016-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nfrom edb.server.protocol cimport frontend\nfrom edb.server.pgproto.pgproto cimport WriteBuffer\n\n\ncdef class NotebookConnection(frontend.AbstractFrontendConnection):\n cdef:\n WriteBuffer buf\n\n cdef bytes _get_data(self)\n" }, { "path": "edb/server/protocol/notebook_ext.pyx", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2019-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nimport base64\nimport http\nimport json\nimport urllib.parse\n\nimport immutables\n\nfrom edb import errors\nfrom edb.server.pgcon import errors as pgerrors\n\nfrom edb.common import debug\nfrom edb.common import markup\n\nfrom edb.server import compiler\nfrom edb.server import defines as edbdef\nfrom edb.server.compiler import OutputFormat\nfrom edb.server.compiler import dbstate\nfrom edb.server.compiler import enums\nfrom edb.server.protocol import execute as p_execute\nfrom edb.server.dbview cimport dbview\nfrom edb.server.protocol cimport frontend\n\nfrom edb.server.pgproto.pgproto cimport (\n WriteBuffer,\n)\n\n\ninclude \"./consts.pxi\"\n\ncdef tuple CURRENT_PROTOCOL = edbdef.CURRENT_PROTOCOL\n\nALLOWED_CAPABILITIES = (\n enums.Capability.MODIFICATIONS |\n enums.Capability.DDL\n)\n\n\ncdef handle_error(\n object request,\n object response,\n error\n):\n if debug.flags.server:\n markup.dump(error)\n\n er_type = type(error)\n if not issubclass(er_type, errors.EdgeDBError):\n er_type = errors.InternalServerError\n\n response.body = json.dumps({\n 'kind': 'error',\n 'error': {\n 'message': str(error),\n 'type': er_type.__name__,\n }\n }).encode()\n response.status = http.HTTPStatus.BAD_REQUEST\n response.close_connection = True\n\n\nasync def handle_request(\n object request,\n object response,\n object db,\n str role_name,\n list args,\n object tenant,\n):\n response.content_type = b'application/json'\n\n if args == ['status'] and request.method == b'GET':\n try:\n await heartbeat_check(db, tenant)\n except Exception as ex:\n return handle_error(request, response, ex)\n else:\n response.status = http.HTTPStatus.OK\n response.body = b'{\"kind\": \"status\", \"status\": \"OK\"}'\n return\n\n if args != []:\n ex = Exception(f'Unknown path')\n return handle_error(request, response, ex)\n\n queries = None\n\n try:\n if request.method == b'POST':\n body = json.loads(request.body)\n if not isinstance(body, dict):\n raise TypeError(\n 'the body of the request must be a JSON object')\n queries = body.get('queries')\n\n else:\n raise TypeError('expected a POST request')\n\n if not queries:\n raise TypeError(\n 'invalid notebook request: \"queries\" is missing')\n\n except Exception as ex:\n return handle_error(request, response, ex)\n\n response.status = http.HTTPStatus.OK\n try:\n result = await execute(db, role_name, tenant, queries)\n except Exception as ex:\n return handle_error(request, response, ex)\n else:\n response.custom_headers['EdgeDB-Protocol-Version'] = \\\n f'{CURRENT_PROTOCOL[0]}.{CURRENT_PROTOCOL[1]}'\n response.custom_headers['Gel-Protocol-Version'] = \\\n f'{CURRENT_PROTOCOL[0]}.{CURRENT_PROTOCOL[1]}'\n response.body = b'{\"kind\": \"results\", \"results\":' + result + b'}'\n\n\nasync def heartbeat_check(db, tenant):\n async with tenant.with_pgcon(db.name) as pgcon:\n await pgcon.sql_execute(b\"SELECT 'OK';\")\n\n\ncdef class NotebookConnection(frontend.AbstractFrontendConnection):\n def __cinit__(self):\n self.buf = WriteBuffer.new()\n\n cdef write(self, WriteBuffer data):\n self.buf.write_bytes(bytes(data))\n\n cdef bytes _get_data(self):\n return bytes(self.buf)\n\n cdef flush(self):\n pass\n\n\nasync def execute(db, role_name, tenant, queries: list):\n dbv: dbview.DatabaseConnectionView = await tenant.new_dbview(\n dbname=db.name,\n query_cache=False,\n protocol_version=edbdef.CURRENT_PROTOCOL,\n role_name=role_name,\n )\n compiler_pool = tenant.server.get_compiler_pool()\n units = await compiler_pool.compile_notebook(\n dbv.dbname,\n dbv.get_user_schema_pickle(),\n dbv.get_global_schema_pickle(),\n dbv.reflection_cache,\n dbv.get_database_config(),\n dbv.get_compilation_system_config(),\n queries,\n CURRENT_PROTOCOL,\n 50, # implicit limit\n client_id=tenant.client_id,\n client_name=tenant.get_instance_name(),\n )\n result = []\n bind_data = None\n async with tenant.with_pgcon(db.name) as pgcon:\n try:\n await pgcon.sql_execute(b'START TRANSACTION;')\n dbv.start_tx()\n\n for is_error, unit_or_error in units:\n if is_error:\n result.append({\n 'kind': 'error',\n 'error': unit_or_error,\n })\n else:\n query_unit = unit_or_error\n query_unit_group = dbstate.QueryUnitGroup()\n query_unit_group.append(query_unit)\n\n dbv.check_capabilities(\n query_unit,\n ALLOWED_CAPABILITIES,\n errors.UnsupportedCapabilityError,\n \"disallowed in notebook\",\n query_unit_group.unsafe_isolation_dangers,\n )\n try:\n if query_unit.in_type_args:\n raise errors.QueryError(\n 'cannot use query parameters in tutorial')\n\n fe_conn = NotebookConnection()\n\n compiled = dbview.CompiledQuery(\n query_unit_group=query_unit_group)\n await p_execute.execute(\n pgcon, dbv, compiled, b'', fe_conn=fe_conn,\n )\n\n except Exception as ex:\n if debug.flags.server:\n markup.dump(ex)\n\n ex = await p_execute.interpret_error(\n ex,\n dbv._db,\n global_schema_pickle=dbv.get_global_schema_pickle(),\n user_schema_pickle=dbv.get_user_schema_pickle(),\n )\n\n result.append({\n 'kind': 'error',\n 'error': [type(ex).__name__, str(ex), {}],\n })\n\n break\n else:\n result.append({\n 'kind': 'data',\n 'data': (\n base64.b64encode(\n query_unit.out_type_id).decode(),\n base64.b64encode(\n query_unit.out_type_data).decode(),\n base64.b64encode(\n fe_conn._get_data()).decode(),\n base64.b64encode(\n query_unit.status).decode(),\n ),\n })\n\n finally:\n try:\n await pgcon.sql_execute(b'ROLLBACK;')\n finally:\n tenant.remove_dbview(dbv)\n\n return json.dumps(result).encode()\n" }, { "path": "edb/server/protocol/pg_ext.pxd", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2022-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nfrom edb.server.pgproto.pgproto cimport WriteBuffer\nfrom edb.server.protocol cimport frontend\nfrom edb.server.pgcon.pgcon cimport PGMessage\ncimport edb.pgsql.parser.parser as pg_parser\n\n\ncdef class PreparedStmt:\n cdef:\n PGMessage parse_action\n pg_parser.Source source\n\n\ncdef class ConnectionView:\n\n cdef:\n object _local_fe_defaults\n object _settings\n object _fe_settings\n\n bint _in_tx_explicit\n bint _in_tx_implicit\n object _in_tx_settings\n object _in_tx_fe_settings\n object _in_tx_fe_local_settings\n dict _in_tx_portals\n object _in_tx_new_portals\n object _in_tx_savepoints\n bint _tx_error\n\n tuple _session_state_db_cache\n\n cdef _init_user_configs(self, username, tenant)\n cpdef inline current_fe_settings(self)\n cdef inline fe_transaction_state(self)\n cpdef inline bint in_tx(self)\n cdef inline _reset_tx_state(\n self, bint chain_implicit, bint chain_explicit\n )\n cdef bint needs_commit_after_state_sync(self)\n cpdef inline close_portal_if_exists(self, str name)\n cpdef inline close_portal(self, str name)\n cdef inline find_portal(self, str name)\n cdef inline portal_exists(self, str name)\n\n\ncdef class PgConnection(frontend.FrontendConnection):\n\n cdef:\n ConnectionView _dbview\n\n bytes secret\n dict prepared_stmts\n dict sql_prepared_stmts\n dict sql_prepared_stmts_map\n dict wrapping_prepared_stmts\n bint ignore_till_sync\n\n object sslctx\n object endpoint_security\n bint is_tls\n bint _disable_cache\n bint _disable_normalization\n\n cdef inline WriteBuffer ready_for_query(self)\n" }, { "path": "edb/server/protocol/pg_ext.pyx", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2022-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nimport codecs\nimport collections\nimport contextlib\nimport copy\nimport encodings.aliases\nimport logging\nimport hashlib\nimport json\nimport os\nimport sys\nimport time\nimport uuid\nfrom collections import deque\nfrom typing import Sequence\n\ncimport cython\nimport immutables\nfrom libc.stdint cimport int32_t, int16_t, uint32_t\n\nfrom edb import errors\nfrom edb.common import debug\nfrom edb.common.log import current_tenant\nfrom edb.pgsql.common import setting_to_sql\nfrom edb.pgsql.parser import exceptions as parser_errors\nimport edb.pgsql.parser.parser as pg_parser\ncimport edb.pgsql.parser.parser as pg_parser\nfrom edb.server import args as srvargs\nfrom edb.server import defines, metrics\nfrom edb.server import tenant as edbtenant\nfrom edb.server.compiler import dbstate, enums\nfrom edb.server.pgcon import errors as pgerror\nfrom edb.server.pgcon.pgcon cimport PGAction, PGMessage\nfrom edb.server.protocol cimport frontend\n\nDEFAULT_SETTINGS = dbstate.DEFAULT_SQL_SETTINGS\nDEFAULT_FE_SETTINGS = dbstate.DEFAULT_SQL_FE_SETTINGS\n\ncdef object logger = logging.getLogger('edb.server')\ncdef object DEFAULT_STATE = None\n\nencodings.aliases.aliases[\"sql_ascii\"] = \"ascii\"\n\n\nclass ExtendedQueryError(Exception):\n pass\n\n\n@contextlib.contextmanager\ndef managed_error():\n try:\n yield\n except Exception as e:\n raise ExtendedQueryError(e)\n\n\n@cython.final\ncdef class PreparedStmt:\n\n def __init__(self, PGMessage parse_action, pg_parser.Source source):\n self.parse_action = parse_action\n self.source = source\n\n\n\n@cython.final\ncdef class ConnectionView:\n def __init__(self):\n self._in_tx_explicit = False\n self._in_tx_implicit = False\n\n # Kepp track of backend settings so that we can sync to use different\n # backend connections (pgcon) within the same frontend connection,\n # see serialize_state() below and its usages in pgcon.pyx.\n self._settings = DEFAULT_SETTINGS\n self._in_tx_settings = None\n\n # Frontend-only settings are defined by the high-level compiler, and\n # tracked only here, syncing between the compiler process,\n # see current_fe_settings(), fe_transaction_state() and usages below.\n self._local_fe_defaults = DEFAULT_FE_SETTINGS\n self._fe_settings = self._local_fe_defaults\n self._in_tx_fe_settings = None\n self._in_tx_fe_local_settings = None\n\n self._in_tx_portals = {}\n self._in_tx_new_portals = set()\n self._in_tx_savepoints = collections.deque()\n self._tx_error = False\n global DEFAULT_STATE\n if DEFAULT_STATE is None:\n DEFAULT_STATE = json.dumps(\n [\n {\n \"type\": \"P\",\n \"name\": key,\n \"value\": setting_to_sql(key, val),\n }\n for key, val in DEFAULT_SETTINGS.items()\n ] + [\n {\n \"type\": \"S\",\n \"name\": key,\n \"value\": setting_to_sql(key, val),\n }\n for key, val in DEFAULT_FE_SETTINGS.items()\n ]\n ).encode(\"utf-8\")\n\n self._session_state_db_cache = (\n DEFAULT_SETTINGS, DEFAULT_FE_SETTINGS, DEFAULT_STATE\n )\n\n cdef _init_user_configs(self, username, tenant):\n assert self._fe_settings is DEFAULT_FE_SETTINGS\n assert self._in_tx_fe_local_settings is None\n role = tenant.get_roles()[username]\n apply_default = role['apply_access_policies_pg_default']\n if apply_default is not None:\n self._local_fe_defaults = self._local_fe_defaults.set(\n 'apply_access_policies_pg',\n (str(apply_default).lower(),),\n )\n self._fe_settings = self._local_fe_defaults\n\n cpdef inline current_fe_settings(self):\n if self.in_tx():\n # For easier access, _in_tx_fe_local_settings is always a superset\n # of _in_tx_fe_settings; _in_tx_fe_settings only keeps track of\n # non-local settings, so that the local settings don't go across\n # transaction boundaries; this must be consistent with dbstate.py.\n return self._in_tx_fe_local_settings or self._local_fe_defaults\n else:\n return self._fe_settings\n\n cdef inline fe_transaction_state(self):\n return dbstate.SQLTransactionState(\n in_tx=self.in_tx(),\n settings=self._fe_settings,\n in_tx_settings=self._in_tx_fe_settings,\n in_tx_local_settings=self._in_tx_fe_local_settings,\n savepoints=[sp[:3] for sp in self._in_tx_savepoints],\n )\n\n cpdef inline bint in_tx(self):\n return self._in_tx_explicit or self._in_tx_implicit\n\n cdef inline _reset_tx_state(\n self, bint chain_implicit, bint chain_explicit\n ):\n # This method is a part of ending a transaction. COMMIT must be handled\n # before calling this method. If any of the chain_* flag is set, a new\n # transaction will be opened immediately after clean-up.\n self._in_tx_implicit = chain_implicit\n self._in_tx_explicit = chain_explicit\n self._in_tx_settings = self._settings if self.in_tx() else None\n self._in_tx_fe_settings = self._fe_settings if self.in_tx() else None\n self._in_tx_fe_local_settings = self._in_tx_fe_settings\n self._in_tx_portals.clear()\n self._in_tx_new_portals.clear()\n self._in_tx_savepoints.clear()\n self._tx_error = False\n\n def start_implicit(self):\n if self._in_tx_implicit:\n raise RuntimeError(\"already in implicit transaction\")\n else:\n if not self.in_tx():\n self._in_tx_settings = self._settings\n self._in_tx_fe_settings = self._fe_settings\n self._in_tx_fe_local_settings = self._fe_settings\n self._in_tx_implicit = True\n\n def end_implicit(self):\n if not self._in_tx_implicit:\n raise RuntimeError(\"not in implicit transaction\")\n if self._in_tx_explicit:\n # There is an explicit transaction, nothing to do other than\n # turning off the implicit flag so that we can start_implicit again\n self._in_tx_implicit = False\n else:\n # Commit or rollback the implicit transaction\n if not self._tx_error:\n self._settings = self._in_tx_settings\n self._fe_settings = self._in_tx_fe_settings\n self._reset_tx_state(False, False)\n\n def on_success(self, unit: dbstate.SQLQueryUnit):\n # Handle ROLLBACK first before self._tx_error\n if unit.tx_action == dbstate.TxAction.ROLLBACK:\n if not self._in_tx_explicit:\n # TODO: warn about \"no tx\" but still rollback implicit\n pass\n self._reset_tx_state(self._in_tx_implicit, unit.tx_chain)\n\n elif unit.tx_action == dbstate.TxAction.ROLLBACK_TO_SAVEPOINT:\n if not self._in_tx_explicit:\n if self._in_tx_implicit:\n self._tx_error = True\n raise errors.TransactionError(\n \"ROLLBACK TO SAVEPOINT can only be used \"\n \"in transaction blocks\"\n )\n while self._in_tx_savepoints:\n (\n sp_name,\n fe_settings,\n fe_local_settings,\n settings,\n new_portals,\n ) = self._in_tx_savepoints[-1]\n for name in new_portals:\n self._in_tx_portals.pop(name, None)\n if sp_name == unit.sp_name:\n new_portals.clear()\n self._in_tx_settings = settings\n self._in_tx_fe_settings = fe_settings\n self._in_tx_fe_local_settings = fe_local_settings\n self._in_tx_new_portals = new_portals\n break\n else:\n self._in_tx_savepoints.pop()\n else:\n self._tx_error = True\n raise errors.TransactionError(\n f'savepoint \"{unit.sp_name}\" does not exist'\n )\n\n elif self._tx_error:\n raise errors.TransactionError(\n \"current transaction is aborted, \"\n \"commands ignored until end of transaction block\"\n )\n\n elif unit.tx_action == dbstate.TxAction.START:\n if self._in_tx_explicit:\n # TODO: warning: there is already a transaction in progress\n pass\n else:\n if not self.in_tx():\n self._in_tx_settings = self._settings\n self._in_tx_fe_settings = self._fe_settings\n self._in_tx_fe_local_settings = self._fe_settings\n self._in_tx_explicit = True\n\n elif unit.tx_action == dbstate.TxAction.COMMIT:\n if not self._in_tx_explicit:\n # TODO: warning: there is no transaction in progress\n # but we still commit implicit transactions if any\n pass\n if self.in_tx():\n self._settings = self._in_tx_settings\n self._fe_settings = self._in_tx_fe_settings\n self._reset_tx_state(self._in_tx_implicit, unit.tx_chain)\n\n elif unit.tx_action == dbstate.TxAction.DECLARE_SAVEPOINT:\n if not self._in_tx_explicit:\n raise errors.TransactionError(\n \"SAVEPOINT can only be used in transaction blocks\"\n )\n self._in_tx_new_portals = set()\n self._in_tx_savepoints.append((\n unit.sp_name,\n self._in_tx_fe_settings,\n self._in_tx_fe_local_settings,\n self._in_tx_settings,\n self._in_tx_new_portals,\n ))\n\n elif unit.tx_action == dbstate.TxAction.RELEASE_SAVEPOINT:\n pass\n\n if unit.set_vars:\n # only session settings here\n if unit.set_vars == {None: None}: # RESET ALL\n if self.in_tx():\n self._in_tx_settings = DEFAULT_SETTINGS\n self._in_tx_fe_settings = self._local_fe_defaults\n self._in_tx_fe_local_settings = self._local_fe_defaults\n else:\n self._settings = DEFAULT_SETTINGS\n self._fe_settings = self._local_fe_defaults\n else:\n if self.in_tx():\n if unit.frontend_only:\n if not unit.is_local:\n settings = self._in_tx_fe_settings.mutate()\n for k, v in unit.set_vars.items():\n if v is None:\n if k in self._local_fe_defaults:\n settings[k] = self._local_fe_defaults[k]\n else:\n settings.pop(k, None)\n else:\n settings[k] = v\n self._in_tx_fe_settings = settings.finish()\n settings = self._in_tx_fe_local_settings.mutate()\n else:\n settings = self._in_tx_settings.mutate()\n elif not unit.is_local:\n if unit.frontend_only:\n settings = self._fe_settings.mutate()\n else:\n settings = self._settings.mutate()\n else:\n return\n for k, v in unit.set_vars.items():\n if v is None:\n if unit.frontend_only and k in self._local_fe_defaults:\n settings[k] = self._local_fe_defaults[k]\n else:\n settings.pop(k, None)\n else:\n settings[k] = v\n if self.in_tx():\n if unit.frontend_only:\n self._in_tx_fe_local_settings = settings.finish()\n else:\n self._in_tx_settings = settings.finish()\n else:\n if unit.frontend_only:\n self._fe_settings = settings.finish()\n else:\n self._settings = settings.finish()\n\n def on_error(self):\n self._tx_error = True\n\n cpdef inline close_portal(self, str name):\n try:\n return self._in_tx_portals.pop(name)\n except KeyError:\n raise pgerror.new(\n pgerror.ERROR_INVALID_CURSOR_NAME,\n f\"cursor \\\"{name}\\\" does not exist\",\n ) from None\n\n cpdef inline close_portal_if_exists(self, str name):\n return self._in_tx_portals.pop(name, None)\n\n def create_portal(self, str name, query_unit):\n if not self.in_tx():\n raise RuntimeError(\n \"portals cannot be created outside a transaction\"\n )\n if name and name in self._in_tx_portals:\n raise pgerror.new(\n pgerror.ERROR_DUPLICATE_CURSOR,\n f\"cursor \\\"{name}\\\" already exists\",\n )\n self._in_tx_portals[name] = query_unit\n\n cdef inline find_portal(self, str name):\n try:\n return self._in_tx_portals[name]\n except KeyError:\n raise pgerror.new(\n pgerror.ERROR_INVALID_CURSOR_NAME,\n f\"cursor \\\"{name}\\\" does not exist\",\n ) from None\n\n cdef inline portal_exists(self, str name):\n return name in self._in_tx_portals\n\n def serialize_state(self):\n if self.in_tx():\n raise errors.InternalServerError(\n 'no need to serialize state while in transaction')\n if (\n self._settings == DEFAULT_SETTINGS\n and self._fe_settings == DEFAULT_FE_SETTINGS\n ):\n return DEFAULT_STATE\n\n if self._session_state_db_cache is not None:\n if self._session_state_db_cache[:2] == (\n self._settings, self._fe_settings\n ):\n return self._session_state_db_cache[-1]\n\n rv = json.dumps(\n [\n {\"type\": \"P\", \"name\": key, \"value\": setting_to_sql(key, val)}\n for key, val in self._settings.items()\n ] + [\n {\"type\": \"S\", \"name\": key, \"value\": setting_to_sql(key, val)}\n for key, val in self._fe_settings.items()\n ]\n ).encode(\"utf-8\")\n self._session_state_db_cache = (self._settings, self._fe_settings, rv)\n return rv\n\n cdef bint needs_commit_after_state_sync(self):\n return any(\n tx_conf in self._settings\n for tx_conf in [\n \"default_transaction_isolation\",\n \"default_transaction_deferrable\",\n \"default_transaction_read_only\",\n ]\n )\n\n\ncdef class PgConnection(frontend.FrontendConnection):\n interface = \"sql\"\n\n def __init__(self, server, sslctx, endpoint_security, **kwargs):\n super().__init__(server, None, **kwargs)\n self._dbview = ConnectionView()\n self._id = str((int(self._id) % (2 ** 32)))\n self.prepared_stmts = {} # via extended query Parse\n self.sql_prepared_stmts = {} # via a PREPARE statement\n self.sql_prepared_stmts_map = {}\n # Tracks prepared statements of operations\n # on *other* prepared statements.\n self.wrapping_prepared_stmts = {}\n self.ignore_till_sync = False\n\n self.sslctx = sslctx\n self.endpoint_security = endpoint_security\n self.is_tls = False\n\n self._disable_cache = debug.flags.disable_qcache\n self._disable_normalization = debug.flags.edgeql_disable_normalization\n\n cdef _main_task_created(self):\n self.server.on_pgext_client_connected(self)\n # complete the client initial message with a mocked type\n self.buffer.feed_data(b'\\xff')\n\n def connection_lost(self, exc):\n self.server.on_pgext_client_disconnected(self)\n super().connection_lost(exc)\n\n cdef is_in_tx(self):\n return self._dbview.in_tx()\n\n cdef write_error(self, exc):\n cdef WriteBuffer buf\n\n if self.debug and not isinstance(exc, errors.BackendUnavailableError):\n self.debug_print('EXCEPTION', type(exc).__name__, exc)\n from edb.common.markup import dump\n dump(exc)\n\n if debug.flags.server and not isinstance(\n exc, errors.BackendUnavailableError\n ):\n self.loop.call_exception_handler({\n 'message': (\n 'an error in edgedb protocol'\n ),\n 'exception': exc,\n 'protocol': self,\n 'transport': self._transport,\n })\n\n message = str(exc)\n\n buf = WriteBuffer.new_message(b'E')\n\n if isinstance(exc, pgerror.BackendError):\n if exc.code_is(pgerror.ERROR_SERIALIZATION_FAILURE):\n metrics.transaction_serialization_errors.inc(\n 1.0, self.get_tenant_label()\n )\n elif isinstance(exc, parser_errors.PSqlUnsupportedError):\n exc = pgerror.FeatureNotSupported(str(exc))\n elif isinstance(exc, parser_errors.PSqlSyntaxError):\n exc = pgerror.new(\n pgerror.ERROR_SYNTAX_ERROR,\n str(exc),\n P=str(exc.cursor_pos),\n )\n elif isinstance(exc, errors.AuthenticationError):\n exc = pgerror.InvalidAuthSpec(str(exc), severity=\"FATAL\")\n elif isinstance(exc, errors.BinaryProtocolError):\n exc = pgerror.ProtocolViolation(\n str(exc), detail=exc.details, severity=\"FATAL\"\n )\n elif isinstance(exc, errors.UnsupportedFeatureError):\n args = {}\n if exc.line >= 0:\n args['L'] = str(exc.line)\n if exc.position >= 0:\n args['P'] = str(exc.position + 1)\n exc = pgerror.new(\n pgerror.ERROR_FEATURE_NOT_SUPPORTED,\n str(exc),\n **args,\n )\n elif isinstance(exc, errors.EdgeDBError):\n args = dict(hint=exc.hint, detail=exc.details)\n if exc.line >= 0:\n args['L'] = str(exc.line)\n if exc.position >= 0:\n # pg uses 1 based indexes for showing errors.\n args['P'] = str(exc.position + 1)\n exc = pgerror.new(\n exc.pgext_code or pgerror.ERROR_INTERNAL_ERROR,\n str(exc),\n **args,\n )\n if isinstance(exc, errors.TransactionSerializationError):\n metrics.transaction_serialization_errors.inc(\n 1.0, self.get_tenant_label()\n )\n else:\n exc = pgerror.new(\n pgerror.ERROR_INTERNAL_ERROR,\n str(exc),\n severity=\"FATAL\",\n )\n\n for k, v in exc.fields.items():\n buf.write_byte(ord(k))\n buf.write_str(v, \"utf-8\")\n buf.write_byte(b'\\0')\n\n self.write(buf.end_message())\n\n async def _handshake(self):\n cdef int16_t proto_ver_major, proto_ver_minor\n\n for first in (True, False):\n if not self.buffer.take_message():\n await self.wait_for_message(report_idling=True)\n\n proto_ver_major = self.buffer.read_int16()\n proto_ver_minor = self.buffer.read_int16()\n if proto_ver_major == 1234:\n if proto_ver_minor == 5678: # CancelRequest\n pid = str(self.buffer.read_int32())\n secret = self.buffer.read_bytes(4)\n self.buffer.finish_message()\n\n if self.debug:\n self.debug_print(\"CancelRequest\", pid, secret)\n self.server.cancel_pgext_connection(pid, secret)\n self.request_stop()\n break\n\n elif proto_ver_minor == 5679: # SSLRequest\n if self.debug:\n self.debug_print(\"SSLRequest\")\n if not first:\n raise pgerror.ProtocolViolation(\n \"found multiple SSLRequest\", severity=\"FATAL\"\n )\n\n self.buffer.finish_message()\n if self._transport is None:\n raise ConnectionAbortedError\n if self.debug:\n self.debug_print(\"S for SSLRequest\")\n self._transport.write(b'S')\n # complete the next client message with a mocked type\n self.buffer.feed_data(b'\\xff')\n self._transport = await self.loop.start_tls(\n self._transport,\n self,\n self.sslctx,\n server_side=True,\n )\n tenant = self.server.retrieve_tenant(\n self._transport.get_extra_info(\"ssl_object\")\n )\n if tenant is edbtenant.host_tenant:\n tenant = None\n self.tenant = tenant\n if self.tenant is not None:\n current_tenant.set(self.tenant.get_instance_name())\n self.is_tls = True\n\n elif proto_ver_minor == 5680: # GSSENCRequest\n raise pgerror.FeatureNotSupported(\n \"GSSENCRequest is not supported\", severity=\"FATAL\"\n )\n\n else:\n raise pgerror.FeatureNotSupported(severity=\"FATAL\")\n\n elif proto_ver_major == 3 and proto_ver_minor == 0:\n # StartupMessage with 3.0 protocol\n if self.debug:\n self.debug_print(\"StartupMessage\")\n if (\n not self.is_tls and self.endpoint_security ==\n srvargs.ServerEndpointSecurityMode.Tls\n ):\n raise pgerror.InvalidAuthSpec(\n \"TLS required due to server endpoint security\",\n severity=\"FATAL\",\n )\n\n await super()._handshake()\n break\n\n else:\n raise pgerror.ProtocolViolation(\n \"invalid protocol version\", severity=\"FATAL\"\n )\n\n def cancel(self, secret):\n if (\n self.secret == secret and\n self._pinned_pgcon is not None and\n not self._pinned_pgcon.idle and\n self.tenant.accept_new_tasks\n ):\n self.tenant.create_task(\n self.tenant.cancel_pgcon_operation(self._pinned_pgcon),\n interruptable=False,\n )\n\n def debug_print(self, *args):\n print(\"::PGEXT::\", f\"id:{self._id}\", *args, file=sys.stderr)\n\n cdef WriteBuffer _make_authentication_sasl_initial(self, list methods):\n cdef WriteBuffer msg_buf\n msg_buf = WriteBuffer.new_message(b'R')\n msg_buf.write_int32(10)\n for method in methods:\n msg_buf.write_bytestring(method)\n msg_buf.write_byte(b'\\0')\n msg_buf.end_message()\n if self.debug:\n self.debug_print(\"AuthenticationSASL:\", *methods)\n return msg_buf\n\n cdef _expect_sasl_initial_response(self):\n mtype = self.buffer.get_message_type()\n if mtype != b'p':\n raise pgerror.ProtocolViolation(\n f'expected SASL response, got message type {mtype}')\n selected_mech = self.buffer.read_null_str()\n try:\n client_first = self.buffer.read_len_prefixed_bytes()\n except BufferError:\n client_first = None\n self.buffer.finish_message()\n if self.debug:\n self.debug_print(\n \"SASLInitialResponse:\",\n selected_mech,\n len(client_first) if client_first else client_first,\n )\n if not client_first:\n # The client didn't send the Client Initial Response\n # in SASLInitialResponse, this is an error.\n raise pgerror.ProtocolViolation(\n 'client did not send the Client Initial Response '\n 'data in SASLInitialResponse')\n return selected_mech, client_first\n\n cdef WriteBuffer _make_authentication_sasl_msg(\n self, bytes data, bint final\n ):\n cdef WriteBuffer msg_buf\n msg_buf = WriteBuffer.new_message(b'R')\n if final:\n msg_buf.write_int32(12)\n else:\n msg_buf.write_int32(11)\n msg_buf.write_bytes(data)\n msg_buf.end_message()\n if self.debug:\n self.debug_print(\n \"AuthenticationSASLFinal\" if final\n else \"AuthenticationSASLContinue\",\n len(data),\n )\n return msg_buf\n\n cdef bytes _expect_sasl_response(self):\n mtype = self.buffer.get_message_type()\n if mtype != b'p':\n raise pgerror.ProtocolViolation(\n f'expected SASL response, got message type {mtype}')\n client_final = self.buffer.consume_message()\n if self.debug:\n self.debug_print(\"SASLResponse\", len(client_final))\n return client_final\n\n def check_readiness(self):\n if self.tenant.is_blocked():\n readiness_reason = self.tenant.get_readiness_reason()\n msg = \"the server is not accepting requests\"\n if readiness_reason:\n msg = f\"{msg}: {readiness_reason}\"\n raise pgerror.CannotConnectNowError(msg)\n elif not self.tenant.is_online():\n readiness_reason = self.tenant.get_readiness_reason()\n msg = \"the server is going offline\"\n if readiness_reason:\n msg = f\"{msg}: {readiness_reason}\"\n raise pgerror.CannotConnectNowError(msg)\n\n async def authenticate(self):\n cdef:\n WriteBuffer msg_buf\n WriteBuffer buf\n\n self.check_readiness()\n\n params = {}\n while True:\n name = self.buffer.read_null_str()\n if not name:\n break\n value = self.buffer.read_null_str()\n params[name.decode(\"utf-8\")] = value.decode(\"utf-8\")\n if self.debug:\n self.debug_print(\"StartupMessage params:\", params)\n if \"user\" not in params:\n raise pgerror.ProtocolViolation(\n \"StartupMessage must have a \\\"user\\\"\", severity=\"FATAL\"\n )\n self.buffer.finish_message()\n\n user = params[\"user\"]\n database = params.get(\"database\", user)\n if \"client_encoding\" in params:\n encoding = params[\"client_encoding\"]\n client_encoding = encodings.normalize_encoding(encoding).upper()\n try:\n codecs.lookup(client_encoding)\n except LookupError:\n raise pgerror.new(\n pgerror.ERROR_INVALID_PARAMETER_VALUE,\n f'invalid value for parameter \"client_encoding\": \"{encoding}\"',\n )\n self._dbview._settings = self._dbview._settings.set(\n \"client_encoding\", (client_encoding,)\n )\n else:\n client_encoding = \"UTF8\"\n\n logger.debug('received pg connection request by %s to database %s',\n user, database)\n\n if database == '__default__':\n database = self.tenant.default_database\n elif (\n database == defines.EDGEDB_OLD_DEFAULT_DB\n and self.tenant.maybe_get_db(\n dbname=defines.EDGEDB_OLD_DEFAULT_DB\n ) is None\n ):\n database = self.tenant.default_database\n\n user = self.tenant.resolve_user_name(user)\n\n await self._authenticate(user, database, params)\n\n logger.debug('successfully authenticated %s in database %s',\n user, database)\n\n if not self.tenant.is_database_connectable(database):\n raise pgerror.InvalidAuthSpec(\n f'database {database!r} does not accept connections',\n severity=\"FATAL\",\n )\n\n self.database = self.tenant.get_db(dbname=database)\n await self.database.introspection()\n\n self.dbname = database\n self.username = user\n\n self._dbview._init_user_configs(user, self.tenant)\n\n buf = WriteBuffer()\n\n msg_buf = WriteBuffer.new_message(b'R')\n msg_buf.write_int32(0)\n msg_buf.end_message()\n buf.write_buffer(msg_buf)\n if self.debug:\n self.debug_print(\"AuthenticationOk\")\n\n self.secret = os.urandom(4)\n msg_buf = WriteBuffer.new_message(b'K')\n msg_buf.write_int32(int(self._id))\n msg_buf.write_bytes(self.secret)\n msg_buf.end_message()\n buf.write_buffer(msg_buf)\n if self.debug:\n self.debug_print(\"BackendKeyData\")\n\n async with self.with_pgcon() as conn:\n for name, value in conn.parameter_status.items():\n msg_buf = WriteBuffer.new_message(b'S')\n msg_buf.write_str(name, \"utf-8\")\n if name == \"client_encoding\":\n value = client_encoding\n elif name == \"server_version\":\n value = str(defines.PGEXT_POSTGRES_VERSION)\n elif name == \"session_authorization\":\n value = user\n elif name == \"application_name\":\n value = self.tenant.get_instance_name()\n msg_buf.write_str(value, \"utf-8\")\n msg_buf.end_message()\n buf.write_buffer(msg_buf)\n if self.debug:\n self.debug_print(f\"ParameterStatus: {name}={value}\")\n self.write(buf)\n # Try to sync the settings, especially client_encoding.\n await conn.sql_apply_state(self._dbview)\n\n self.write(self.ready_for_query())\n self.flush()\n\n cdef inline WriteBuffer ready_for_query(self):\n cdef WriteBuffer msg_buf\n self.ignore_till_sync = False\n msg_buf = WriteBuffer.new_message(b'Z')\n if self._dbview.in_tx():\n if self._dbview._tx_error:\n msg_buf.write_byte(b'E')\n else:\n msg_buf.write_byte(b'T')\n else:\n msg_buf.write_byte(b'I')\n return msg_buf.end_message()\n\n def on_success(self, query_unit):\n cdef:\n PreparedStmt stmt\n\n if query_unit.deallocate is not None:\n stmt_name = query_unit.deallocate.stmt_name\n self.sql_prepared_stmts.pop(stmt_name, None)\n self.sql_prepared_stmts_map.pop(stmt_name, None)\n self.prepared_stmts.pop(stmt_name, None)\n # If any wrapping prepared statements referred to this\n # prepared statement, invalidate them.\n for wrapping_ps in self.wrapping_prepared_stmts.pop(stmt_name, []):\n stmt = self.prepared_stmts.get(wrapping_ps)\n if stmt is not None:\n stmt.parse_action.invalidate()\n\n def on_error(self, query_unit):\n cdef:\n PreparedStmt stmt\n\n if query_unit.prepare is not None:\n stmt_name = query_unit.prepare.stmt_name\n self.sql_prepared_stmts.pop(stmt_name, None)\n self.sql_prepared_stmts_map.pop(stmt_name, None)\n self.prepared_stmts.pop(stmt_name, None)\n # If any wrapping prepared statements referred to this\n # prepared statement, invalidate them.\n for wrapping_ps in self.wrapping_prepared_stmts.pop(stmt_name, []):\n stmt = self.prepared_stmts.get(wrapping_ps)\n if stmt is not None:\n stmt.parse_action.invalidate()\n\n async def main_step(self, char mtype):\n try:\n await self._main_step(mtype)\n except pgerror.BackendError as ex:\n self.write_error(ex)\n self.write(self.ready_for_query())\n self.flush()\n self.request_stop()\n\n async def _main_step(self, char mtype):\n cdef:\n WriteBuffer buf\n ConnectionView dbv\n\n dbv = self._dbview\n\n self.check_readiness()\n\n if self.debug:\n self.debug_print(\"main_step\", repr(chr(mtype)))\n if self.ignore_till_sync:\n self.debug_print(\"ignoring\")\n\n if mtype == b'S': # Sync\n self.buffer.finish_message()\n if self.debug:\n self.debug_print(\"Sync\")\n if dbv._in_tx_implicit:\n actions = [PGMessage(PGAction.SYNC)]\n async with self.with_pgcon() as conn:\n success, _ = await conn.sql_extended_query(\n actions, self, self.database.dbver, dbv)\n self.ignore_till_sync = not success\n else:\n self.ignore_till_sync = False\n self.write(self.ready_for_query())\n self.flush()\n\n elif mtype == b'X': # Terminate\n self.buffer.finish_message()\n if self.debug:\n self.debug_print(\"Terminate\")\n self.close()\n return True\n\n elif self.ignore_till_sync:\n self.buffer.discard_message()\n\n elif mtype == b'Q': # Query\n try:\n query = self.buffer.read_null_str()\n metrics.query_size.observe(\n len(query), self.get_tenant_label(), 'sql'\n )\n query_str = query.decode(\"utf8\")\n self.buffer.finish_message()\n if self.debug:\n self.debug_print(\"Query\", query_str)\n actions = await self.simple_query(query_str)\n del query_str, query\n except Exception as ex:\n self.write_error(ex)\n self.write(self.ready_for_query())\n self.flush()\n\n else:\n async with self.with_pgcon() as conn:\n try:\n _, rq_sent = await conn.sql_extended_query(\n actions,\n self,\n self.database.dbver,\n dbv,\n )\n except Exception as ex:\n self.write_error(ex)\n self.write(self.ready_for_query())\n else:\n if not rq_sent:\n self.write(self.ready_for_query())\n\n self.flush()\n\n elif (\n mtype == b'P' or mtype == b'B' or mtype == b'D' or mtype == b'E' or\n # One of Parse, Bind, Describe or Execute starts an extended query\n mtype == b'C' # or Close\n ):\n try:\n actions, exception = await self.extended_query()\n except ExtendedQueryError as ex:\n actions = ()\n exception = ex\n else:\n async with self.with_pgcon() as conn:\n try:\n success, _ = await conn.sql_extended_query(\n actions, self, self.database.dbver, dbv)\n self.ignore_till_sync = not success\n except Exception as ex:\n self.write_error(ex)\n self.flush()\n self.ignore_till_sync = True\n if exception:\n self.write_error(exception.args[0])\n self.flush()\n self.ignore_till_sync = True\n\n elif mtype == b'H': # Flush\n self.buffer.finish_message()\n if self.debug:\n self.debug_print(\"Flush\")\n self.flush()\n\n else:\n if self.debug:\n self.debug_print(\n \"MESSAGE\", chr(mtype), self.buffer.consume_message()\n )\n raise pgerror.FeatureNotSupported()\n\n async def simple_query(self, query_str: str) -> list[PGMessage]:\n cdef:\n PreparedStmt stmt\n\n actions = []\n dbv = self._dbview\n\n if self._disable_normalization:\n source = pg_parser.Source.from_string(query_str)\n else:\n source = pg_parser.NormalizedSource.from_string(query_str)\n query_units = await self.compile(source, dbv)\n\n # TODO: currently, normalization does not work with multiple queries\n # so we must re-run the compilation with non-normalized query.\n # Ideally we could detect this before compilation.\n if len(query_units) > 1:\n source = pg_parser.Source.from_string(query_str)\n query_units = await self.compile(source, dbv)\n\n for qu in query_units:\n self.check_capabilities(qu)\n\n already_in_implicit_tx = dbv._in_tx_implicit\n metrics.sql_queries.inc(\n len(query_units), self.tenant.get_instance_name()\n )\n self._query_count += len(query_units)\n\n if not already_in_implicit_tx:\n actions.append(PGMessage(PGAction.START_IMPLICIT_TX))\n\n for qu in query_units:\n if qu.execute is not None:\n fe_settings = dbv.current_fe_settings()\n known_be_name = (\n self.sql_prepared_stmts_map.get(qu.execute.stmt_name))\n recompile = (\n qu.fe_settings != fe_settings\n or qu.execute.be_stmt_name != known_be_name.encode(\"utf-8\")\n )\n actions.extend(await self._ensure_nested_ps_exists(\n dbv, qu, force_recompilation=recompile))\n else:\n recompile = False\n if recompile:\n stmt, new_stmts = await self._parse_statement(\n stmt_name=None,\n query_str=qu.orig_query,\n parse_data=b\"\\x00\\x00\",\n dbv=dbv,\n force_recompilation=True,\n injected_action=True,\n )\n else:\n stmt, new_stmts = await self._parse_unit(\n stmt_name=None,\n unit=qu,\n source=source,\n parse_data=b\"\\x00\\x00\",\n dbv=dbv,\n injected_action=True,\n )\n parse_unit = stmt.parse_action.query_unit\n if parse_unit.set_vars:\n actions.extend(self._build_sql_settings_actions(parse_unit))\n actions.append(stmt.parse_action)\n\n # 2 bytes: number of format codes (1)\n # 2 bytes: first format code (1) is binary\n # (this implies that all args are binary)\n # 2 bytes: number of arguments (0)\n # 2 bytes: number of result format codes (0)\n # (this implies that )\n bind_data = b\"\\x00\\x01\\x00\\x01\\x00\\x00\\x00\\x00\"\n # remap argumnets, which will inject globals\n bind_data = remap_arguments(\n bind_data,\n parse_unit.params,\n dbv.current_fe_settings(),\n source,\n self.get_permissions(),\n self.username,\n )\n actions.append(\n PGMessage(\n PGAction.BIND,\n portal_name=\"\",\n stmt_name=parse_unit.stmt_name,\n args=bind_data,\n query_unit=parse_unit,\n injected=True,\n )\n )\n\n actions.append(\n PGMessage(\n PGAction.DESCRIBE_STMT_ROWS,\n stmt_name=parse_unit.stmt_name,\n query_unit=parse_unit,\n )\n )\n actions.append(\n PGMessage(\n PGAction.EXECUTE,\n args=0,\n portal_name=\"\",\n query_unit=parse_unit,\n injected=False,\n )\n )\n actions.append(\n PGMessage(\n PGAction.CLOSE_PORTAL,\n portal_name=\"\",\n query_unit=parse_unit,\n injected=True,\n )\n )\n\n actions.append(PGMessage(PGAction.SYNC))\n\n return actions\n\n async def extended_query(self):\n cdef:\n WriteBuffer buf\n int16_t i\n bytes data\n bint in_implicit\n PreparedStmt stmt\n ConnectionView dbv\n\n # Extended-query pre-plays on a deeply-cloned temporary dbview so as to\n # compose the actions list with correct states; the actual changes to\n # dbview is applied in pgcon.pyx when the actions are actually executed\n dbv = copy.deepcopy(self._dbview)\n\n actions = deque()\n fresh_stmts = set()\n in_implicit = self._dbview._in_tx_implicit\n\n # Here we will exhaust the buffer and queue up actions for the backend.\n # Any error in this step will be handled in the outer main_step() -\n # the error will be returned, any remaining messages in the buffer will\n # be discarded until a Sync message is found (ignore_till_sync).\n # This also means no partial action is executed in the backend for now.\n while self.buffer.take_message():\n if not in_implicit:\n actions.append(PGMessage(PGAction.START_IMPLICIT_TX))\n in_implicit = True\n with managed_error():\n dbv.start_implicit()\n mtype = self.buffer.get_message_type()\n\n if mtype == b'P': # Parse\n stmt_name = self.buffer.read_null_str().decode(\"utf8\")\n query_bytes = self.buffer.read_null_str()\n query_str = query_bytes.decode(\"utf8\")\n data = self.buffer.consume_message()\n if self.debug:\n self.debug_print(\"Parse\", repr(stmt_name), query_str, data)\n metrics.query_size.observe(\n len(query_bytes), self.get_tenant_label(), 'sql'\n )\n\n with managed_error():\n if (\n stmt_name and (\n stmt_name in self.prepared_stmts\n or stmt_name in self.sql_prepared_stmts\n )\n ):\n raise pgerror.new(\n pgerror.ERROR_DUPLICATE_PREPARED_STATEMENT,\n f\"prepared statement \\\"{stmt_name}\\\" already \"\n f\"exists\",\n )\n\n stmt, new_stmts = await self._parse_statement(\n stmt_name, query_str, data, dbv\n )\n if stmt.parse_action.query_unit.execute is not None:\n actions.extend(\n await self._ensure_nested_ps_exists(\n dbv,\n stmt.parse_action.query_unit,\n )\n )\n fresh_stmts.update(new_stmts)\n actions.append(stmt.parse_action)\n\n elif mtype == b'B': # Bind\n portal_name = self.buffer.read_null_str().decode(\"utf8\")\n stmt_name = self.buffer.read_null_str().decode(\"utf8\")\n data = self.buffer.consume_message()\n if self.debug:\n self.debug_print(\n \"Bind\", repr(portal_name), repr(stmt_name), data\n )\n\n with managed_error():\n stmt = await self._ensure_ps_locality(\n dbv,\n stmt_name,\n fresh_stmts,\n actions,\n )\n\n try:\n params = stmt.parse_action.query_unit.params\n fe_settings = dbv.current_fe_settings()\n data = remap_arguments(\n data,\n params,\n fe_settings,\n stmt.source,\n self.get_permissions(),\n self.username,\n )\n except Exception as e:\n # we return here instead of raising the exception\n # because we want to also return the previous actions\n return actions, ExtendedQueryError(e)\n\n actions.append(\n PGMessage(\n PGAction.BIND,\n stmt_name=stmt.parse_action.stmt_name,\n portal_name=portal_name,\n args=data,\n query_unit=stmt.parse_action.query_unit,\n )\n )\n dbv.create_portal(portal_name, stmt.parse_action.query_unit)\n\n elif mtype == b'D': # Describe\n kind = self.buffer.read_byte()\n name = self.buffer.read_null_str().decode(\"utf8\")\n self.buffer.finish_message()\n if self.debug:\n self.debug_print(\"Describe\", kind, repr(name))\n\n with managed_error():\n if kind == b'S': # prepared statement\n stmt = await self._ensure_ps_locality(\n dbv,\n name,\n fresh_stmts,\n actions,\n )\n actions.append(\n PGMessage(\n PGAction.DESCRIBE_STMT,\n stmt_name=stmt.parse_action.stmt_name,\n query_unit=stmt.parse_action.query_unit,\n )\n )\n\n elif kind == b'P': # portal\n actions.append(\n PGMessage(\n PGAction.DESCRIBE_PORTAL,\n portal_name=name,\n query_unit=dbv.find_portal(name),\n )\n )\n\n else:\n raise pgerror.ProtocolViolation(\n \"invalid Describe kind\"\n )\n\n elif mtype == b'E': # Execute\n portal_name = self.buffer.read_null_str().decode(\"utf8\")\n max_rows = self.buffer.read_int32()\n self.buffer.finish_message()\n if self.debug:\n self.debug_print(\"Execute\", repr(portal_name), max_rows)\n\n metrics.sql_queries.inc(1.0, self.tenant.get_instance_name())\n self._query_count += 1\n with managed_error():\n unit = dbv.find_portal(portal_name)\n if unit.set_vars:\n actions.extend(self._build_sql_settings_actions(unit))\n actions.append(\n PGMessage(\n PGAction.EXECUTE,\n portal_name=portal_name,\n args=max_rows,\n query_unit=unit,\n )\n )\n dbv.on_success(unit)\n\n elif mtype == b'C': # Close\n kind = self.buffer.read_byte()\n name_bytes = self.buffer.read_null_str()\n name = name_bytes.decode(\"utf8\")\n self.buffer.finish_message()\n if self.debug:\n self.debug_print(\"Close\", kind, repr(name))\n\n with managed_error():\n if kind == b'S': # prepared statement\n if name not in self.prepared_stmts:\n raise pgerror.new(\n pgerror.ERROR_INVALID_SQL_STATEMENT_NAME,\n f\"prepared statement \\\"{name}\\\" does not \"\n f\"exist\",\n )\n # The prepared statement in the backend is managed by\n # the LRU cache in pgcon.pyx, we don't close it here\n fresh_stmts.discard(name)\n self.prepared_stmts.pop(name)\n actions.append(\n PGMessage(\n PGAction.CLOSE_STMT,\n stmt_name=name_bytes,\n ),\n )\n\n elif kind == b'P': # portal\n actions.append(\n PGMessage(\n PGAction.CLOSE_PORTAL,\n portal_name=name,\n query_unit=dbv.close_portal(name),\n ),\n )\n\n else:\n raise pgerror.ProtocolViolation(\"invalid Close kind\")\n\n elif mtype == b'H': # Flush\n self.buffer.finish_message()\n if self.debug:\n self.debug_print(\"Flush\")\n actions.append(PGMessage(PGAction.FLUSH))\n\n elif mtype == b'S': # Sync\n in_implicit = False\n self.buffer.finish_message()\n if self.debug:\n self.debug_print(\"Sync\")\n with managed_error():\n actions.append(PGMessage(PGAction.SYNC))\n dbv.end_implicit()\n break\n\n else:\n # Other messages would cut off the current extended_query()\n break\n\n if self.debug:\n self.debug_print(\"extended_query\", actions)\n return actions, None\n\n def check_capabilities(\n self,\n query_unit,\n ):\n query_capabilities = query_unit.capabilities\n\n role_capability = self.get_role_capability()\n if query_capabilities & ~role_capability:\n raise query_capabilities.make_error(\n role_capability,\n errors.DisabledCapabilityError,\n f\"role {self.username} does not have permission\",\n )\n\n def get_role_capability(self) -> enums.Capability:\n if capability := self.tenant.get_role_capabilities().get(\n self.username\n ):\n return capability\n return enums.Capability.NONE\n\n def get_permissions(self) -> tuple[bool, Sequence[str]]:\n if role_desc := self.tenant.get_roles().get(self.username):\n return (\n bool(role_desc.get('superuser')),\n (role_desc.get('all_permissions') or ())\n )\n return False, ()\n\n async def _ensure_ps_locality(\n self,\n dbv: ConnectionView,\n stmt_name: str,\n local_stmts: set[str],\n actions\n ) -> PreparedStmt:\n \"\"\"Make sure given *stmt_name* is known by Postgres\n\n Frontend SQL connections do not normally own Postgres connections,\n so there is no affinity between them. Thus, whenever we receive\n a message operating on some prepared statement, we must ensure\n that this statement has been prepared in the currently active\n Postgres connection. We rely on pgcon LRU to actually make a\n decision on whether to issue the injected Parse messages.\n\n NB: this method mutates *local_stmts* and *actions*.\n \"\"\"\n cdef:\n PreparedStmt stmt\n\n stmt = self.prepared_stmts.get(stmt_name)\n if stmt is None:\n raise pgerror.new(\n pgerror.ERROR_INVALID_SQL_STATEMENT_NAME,\n f\"prepared statement \\\"{stmt_name}\\\" does not \"\n f\"exist\",\n )\n\n if stmt_name not in local_stmts:\n # Non-local statement, so inject its Parse.\n fe_settings = dbv.current_fe_settings()\n qu = stmt.parse_action.query_unit\n assert qu is not None\n if stmt.parse_action.fe_settings != fe_settings:\n # Some of the statically compiler-evaluated\n # queries like `current_schema` depend on the\n # fe_settings, we need to re-compile if the\n # fe_settings have changed.\n stmt.parse_action.invalidate()\n\n if (\n qu.execute is not None\n and (\n qu.execute.be_stmt_name\n != self.sql_prepared_stmts_map.get(\n qu.execute.stmt_name).encode(\"utf-8\")\n )\n ):\n # Likewise, re-compile if this is an EXECUTE query\n # and the translated name of the prepared statement\n # has changed (e.g. due to it having been deallocated\n # and prepared with a different query).\n stmt.parse_action.invalidate()\n\n if not stmt.parse_action.is_valid():\n parse_actions, new_stmts = await self._reparse(\n stmt_name,\n stmt.parse_action,\n dbv,\n )\n local_stmts.update(new_stmts)\n actions.extend(parse_actions)\n stmt = self.prepared_stmts[stmt_name]\n else:\n actions.append(stmt.parse_action.as_injected())\n local_stmts.add(stmt_name)\n\n return stmt\n\n async def _reparse(\n self,\n str stmt_name,\n PGMessage parse_action,\n ConnectionView dbv,\n ):\n cdef:\n PreparedStmt outer_stmt\n actions = []\n qu = parse_action.query_unit\n assert qu is not None\n\n if self.debug:\n self.debug_print(\"reparsing\", stmt_name, parse_action)\n\n if (\n qu.prepare is not None\n or qu.execute is not None\n ):\n actions.extend(\n await self._ensure_nested_ps_exists(\n dbv,\n qu,\n force_recompilation=True,\n ),\n )\n\n outer_stmt, new_stmts = await self._parse_statement(\n stmt_name,\n qu.orig_query,\n parse_action.args[2],\n dbv,\n force_recompilation=True,\n injected_action=True,\n )\n\n actions.append(outer_stmt.parse_action)\n\n return actions, new_stmts\n\n async def _ensure_nested_ps_exists(\n self,\n dbv: ConnectionView,\n execute_unit: dbstate.SQLQueryUnit,\n force_recompilation: bool = False,\n ) -> list[PGMessage]:\n cdef:\n PreparedStmt sql_stmt\n\n exec_data = execute_unit.execute\n prep_qu = self.sql_prepared_stmts.pop(exec_data.stmt_name, None)\n actions = []\n\n if prep_qu is None:\n raise pgerror.new(\n pgerror.ERROR_INVALID_SQL_STATEMENT_NAME,\n f\"prepared statement \"\n f\"\\\"{exec_data.stmt_name}\\\" does not \"\n f\"exist\",\n )\n\n sql_stmt, _ = await self._parse_statement(\n prep_qu.stmt_name.decode(\"utf-8\"),\n prep_qu.orig_query,\n b\"\\x00\\x00\",\n dbv,\n injected_action=True,\n force_recompilation=force_recompilation,\n )\n actions.append(sql_stmt.parse_action)\n parse_stmt_name = sql_stmt.parse_action.stmt_name\n portal_name = parse_stmt_name.decode(\"utf-8\")\n parse_query_unit = sql_stmt.parse_action.query_unit\n actions.append(\n PGMessage(\n PGAction.BIND,\n portal_name=portal_name,\n stmt_name=parse_stmt_name,\n args=b\"\\x00\\x01\\x00\\x01\\x00\\x00\\x00\\x00\",\n query_unit=parse_query_unit,\n injected=True,\n )\n )\n actions.append(\n PGMessage(\n PGAction.EXECUTE,\n args=0,\n portal_name=portal_name,\n query_unit=parse_query_unit,\n injected=True,\n )\n )\n actions.append(\n PGMessage(\n PGAction.CLOSE_PORTAL,\n portal_name=portal_name,\n query_unit=parse_query_unit,\n injected=True,\n )\n )\n return actions\n\n def _build_sql_settings_actions(self, qu):\n actions = []\n if qu.set_vars == {None: None}: # RESET ALL\n actions.append(\n PGMessage(\n PGAction.BIND,\n force_portal_name=b\"injected\",\n stmt_name=b\"_reset_sql_state_all\",\n # 2 bytes: number of format codes (0)\n # 2 bytes: number of parameters (0)\n # 2 bytes: number of result format codes (0)\n args=b\"\\x00\\x00\\x00\\x00\\x00\\x00\",\n injected=True,\n )\n )\n actions.append(\n PGMessage(\n PGAction.EXECUTE,\n args=0,\n force_portal_name=b\"injected\",\n injected=True,\n )\n )\n actions.append(\n PGMessage(\n PGAction.CLOSE_PORTAL,\n force_portal_name=b\"injected\",\n injected=True,\n )\n )\n elif qu.frontend_only:\n for k, v in qu.set_vars.items():\n buf = WriteBuffer.new()\n\n buf.write_int16(0) # number of format codes\n\n # number of parameters:\n if v is None:\n buf.write_int16(2)\n else:\n buf.write_int16(3)\n\n buf.write_len_prefixed_utf8(k) # 1st param: name\n buf.write_len_prefixed_utf8( # 2nd param: type\n \"L\" if qu.is_local else \"S\")\n if v is not None:\n buf.write_len_prefixed_utf8( # 3rd param: value\n setting_to_sql(k, v))\n\n buf.write_int16(0) # number of result format codes\n\n if v is None:\n actions.append(\n PGMessage(\n PGAction.BIND,\n force_portal_name=b\"injected\",\n stmt_name=b\"_reset_sql_state\",\n args=bytes(buf),\n injected=True,\n )\n )\n else:\n actions.append(\n PGMessage(\n PGAction.BIND,\n force_portal_name=b\"injected\",\n stmt_name=b\"_set_sql_state\",\n args=bytes(buf),\n injected=True,\n )\n )\n actions.append(\n PGMessage(\n PGAction.EXECUTE,\n args=0,\n force_portal_name=b\"injected\",\n injected=True,\n )\n )\n actions.append(\n PGMessage(\n PGAction.CLOSE_PORTAL,\n force_portal_name=b\"injected\",\n injected=True,\n )\n )\n return actions\n\n async def _parse_statement(\n self,\n stmt_name: str | None,\n query_str: str,\n parse_data: bytes,\n dbv: ConnectionView,\n force_recompilation: bool = False,\n injected_action: bool = False,\n ) -> Tuple[PreparedStmt, set[str]]:\n \"\"\"Generate a PARSE action for *query_str*.\n\n The *query_str* string must contain exactly one SQL statement.\n \"\"\"\n stmts = set()\n\n if self._disable_normalization:\n source = pg_parser.Source.from_string(query_str)\n else:\n source = pg_parser.NormalizedSource.from_string(query_str)\n\n query_units = await self.compile(\n source, dbv, ignore_cache=force_recompilation\n )\n if len(query_units) > 1:\n raise pgerror.new(\n pgerror.ERROR_SYNTAX_ERROR,\n \"cannot insert multiple commands into a prepared \"\n \"statement\",\n )\n\n for qu in query_units:\n self.check_capabilities(qu)\n\n return await self._parse_unit(\n stmt_name,\n query_units[0],\n source,\n parse_data,\n dbv,\n injected_action=injected_action,\n )\n\n async def _parse_unit(\n self,\n stmt_name: str | None,\n unit: dbstate.SQLQueryUnit,\n source: pg_parser.Source,\n parse_data: bytes,\n dbv: ConnectionView,\n injected_action: bool = False,\n ) -> Tuple[PreparedStmt, set[str]]:\n stmts = set()\n\n fe_settings = dbv.current_fe_settings()\n nested_ps_name = None\n if unit.prepare is not None:\n # Statement-level PREPARE\n nested_ps_name = unit.prepare.stmt_name\n unit = self._validate_prepare_stmt(unit)\n stmts.add(nested_ps_name)\n self.sql_prepared_stmts[nested_ps_name] = unit\n self.sql_prepared_stmts_map[nested_ps_name] = (\n unit.prepare.be_stmt_name.decode(\"utf-8\"))\n elif unit.execute is not None:\n # Statement-level EXECUTE\n nested_ps_name = unit.execute.stmt_name\n unit = self._validate_execute_stmt(unit)\n elif unit.deallocate is not None:\n # Statement-level DEALLOCATE\n nested_ps_name = unit.deallocate.stmt_name\n unit = self._validate_deallocate_stmt(unit)\n\n remapped_parse_data = remap_parameters(parse_data, unit.params)\n\n action = PGMessage(\n PGAction.PARSE,\n stmt_name=unit.stmt_name,\n args=(unit.query.encode(\"utf-8\"), remapped_parse_data, parse_data),\n query_unit=unit,\n fe_settings=fe_settings,\n injected=injected_action,\n )\n\n if stmt_name is not None and nested_ps_name is not None:\n # This is a prepared statement of an operation on *another*\n # prepared statement, and so we must track this relationship\n # in case the nested prepared statement gets deallocated.\n try:\n self.wrapping_prepared_stmts[nested_ps_name].add(stmt_name)\n except KeyError:\n self.wrapping_prepared_stmts[nested_ps_name] = set([stmt_name])\n\n stmt = PreparedStmt(\n parse_action=action,\n source=source,\n )\n\n if stmt_name is not None:\n self.prepared_stmts[stmt_name] = stmt\n stmts.add(stmt_name)\n\n return stmt, stmts\n\n async def compile(\n self, source: pg_parser.Source, ConnectionView dbv, ignore_cache=False\n ) -> List[dbstate.SQLQueryUnit]:\n if self.debug:\n self.debug_print(\"Compile\", source.text())\n\n fe_settings = dbv.current_fe_settings()\n key = compute_cache_key(source, fe_settings)\n\n ignore_cache |= self._disable_cache\n\n result: List[dbstate.SQLQueryUnit]\n if not ignore_cache:\n result = self.database.lookup_compiled_sql(key)\n if result is not None:\n return result\n # Remember the schema version we are compiling on, so that we can\n # cache the result with the matching version. In case of concurrent\n # schema update, we're only storing an outdated cache entry, and\n # the next identical query could get recompiled on the new schema.\n schema_version = self.database.schema_version\n compiler_pool = self.server.get_compiler_pool()\n started_at = time.monotonic()\n try:\n result = await compiler_pool.compile_sql(\n self.dbname,\n self.database.user_schema_pickle,\n self.database._index._global_schema_pickle,\n self.database.reflection_cache,\n self.database.db_config,\n self.database._index.get_compilation_system_config(),\n source,\n dbv.fe_transaction_state(),\n self.sql_prepared_stmts_map,\n self.dbname,\n self.username,\n client_id=self.tenant.client_id,\n client_name=self.tenant.get_instance_name(),\n )\n finally:\n metrics.query_compilation_duration.observe(\n time.monotonic() - started_at,\n self.tenant.get_instance_name(),\n \"sql\",\n )\n self.database.cache_compiled_sql(key, result, schema_version)\n metrics.sql_compilations.inc(\n len(result), self.tenant.get_instance_name()\n )\n if self.debug:\n self.debug_print(\"Compile result\", result)\n return result\n\n def _validate_prepare_stmt(self, qu):\n assert qu.prepare is not None\n stmt_name = qu.prepare.stmt_name\n if (\n stmt_name in self.prepared_stmts\n or stmt_name in self.sql_prepared_stmts\n ):\n raise pgerror.new(\n pgerror.ERROR_DUPLICATE_PREPARED_STATEMENT,\n f\"prepared statement \\\"{stmt_name}\\\" \"\n f\"already exists\",\n )\n return qu\n\n def _validate_execute_stmt(self, qu):\n assert qu.execute is not None\n stmt_name = qu.execute.stmt_name\n sql_ps = self.sql_prepared_stmts.get(stmt_name)\n if sql_ps is None:\n raise pgerror.new(\n pgerror.ERROR_INVALID_SQL_STATEMENT_NAME,\n f\"prepared statement \\\"{stmt_name}\\\" does \"\n f\"not exist\",\n )\n return qu\n\n def _validate_deallocate_stmt(self, qu):\n assert qu.deallocate is not None\n stmt_name = qu.deallocate.stmt_name\n sql_ps = self.sql_prepared_stmts.get(stmt_name)\n if sql_ps is None:\n raise pgerror.new(\n pgerror.ERROR_INVALID_SQL_STATEMENT_NAME,\n f\"prepared statement \\\"{stmt_name}\\\" does \"\n f\"not exist\",\n )\n return qu\n\n\ndef compute_cache_key(\n source: pg_parser.Source, fe_settings: dbstate.SQLSettings\n) -> bytes:\n h = hashlib.blake2b(source.cache_key())\n for key, value in fe_settings.items():\n if key.startswith('global '):\n continue\n h.update(hash(value).to_bytes(8, signed=True))\n return h.digest()\n\n\ncdef bytes remap_arguments(\n data: bytes,\n params: list[dbstate.SQLParam] | None,\n fe_settings: dbstate.SQLSettings,\n source: pg_parser.Source,\n permission_info: tuple[bool, Sequence[str]],\n username: str, # HACK\n):\n cdef:\n int16_t param_format_count\n int32_t offset\n int32_t arg_offset_external\n int16_t param_count_external\n int32_t size\n\n is_superuser, permissions = permission_info\n\n # The \"external\" parameters (that are visible to the user)\n # don't include the internal params for globals and extracted constants.\n\n # So when we send external params to postgres, we remap them\n # to correct positions and add the globals.\n\n buf = WriteBuffer.new()\n\n # remap param format codes\n param_format_count = read_int16(data[0:2])\n offset = 2\n if params:\n buf.write_int16(len(params))\n for i, param in enumerate(params):\n if isinstance(param, dbstate.SQLParamExternal):\n if param_format_count == 0:\n buf.write_int16(0) # text\n elif param_format_count == 1:\n buf.write_bytes(\n data[offset:offset+2]\n )\n else:\n o = offset + i * 2\n buf.write_bytes(data[o:o+2])\n elif isinstance(param, dbstate.SQLParamExtractedConst):\n buf.write_int16(0) # text\n else:\n # this is for globals\n buf.write_int16(1) # binary\n else:\n buf.write_int16(0)\n offset += param_format_count * 2\n\n # find positions of external args\n arg_count_external = read_int16(data[offset:offset+2])\n offset += 2\n arg_offset_external = offset\n for p in range(arg_count_external):\n size = read_int32(data[offset:offset+4])\n if size == -1: # special case: NULL\n size = 0\n size += 4 # for size which is int32\n offset += size\n\n # write remapped args\n if params:\n buf.write_int16(len(params))\n\n param_count_external = 0\n for i, param in enumerate(params):\n if not isinstance(param, dbstate.SQLParamExternal):\n break\n param_count_external = i + 1\n if param_count_external != arg_count_external:\n raise pgerror.new(\n pgerror.ERROR_PROTOCOL_VIOLATION,\n f'bind message supplies {arg_count_external} '\n f'parameters, but prepared statement \"\" requires '\n f'{param_count_external}',\n )\n\n # write external args\n if arg_offset_external < offset:\n buf.write_bytes(data[arg_offset_external:offset])\n\n # write non-external args\n extracted_consts = list(source.variables().values())\n for (e, param) in enumerate(params[param_count_external:]):\n if isinstance(param, dbstate.SQLParamExtractedConst):\n buf.write_len_prefixed_bytes(extracted_consts[e])\n elif isinstance(param, dbstate.SQLParamGlobal):\n name = param.global_name\n if param.is_permission:\n buf.write_int32(1)\n buf.write_byte(is_superuser or str(name) in permissions)\n elif name.module == 'sys' and name.name == 'current_role':\n write_arg(buf, param.pg_type, (username,))\n else:\n setting_name = f'global {name.module}::{name.name}'\n values = fe_settings.get(setting_name, None)\n\n if values == None:\n buf.write_int32(-1) # NULL\n else:\n write_arg(buf, param.pg_type, values)\n else:\n buf.write_int16(0)\n\n # result format codes\n buf.write_bytes(data[offset:])\n return bytes(buf)\n\n\ncdef bytes remap_parameters(\n data: bytes,\n params: list[dbstate.SQLParam] | None\n):\n # Inject parameter type descriptions in parse message for parameters for\n # globals and extracted constants.\n\n if not params:\n return b\"\\x00\\x00\"\n\n buf = WriteBuffer.new()\n buf.write_int16(len(params))\n\n # copy the params specified by user\n specified_ext = read_int16(data[0:2])\n buf.write_bytes(data[2:2 + specified_ext*4])\n\n for index, param in enumerate(params):\n\n # already written\n if index < specified_ext:\n continue\n\n if isinstance(param, dbstate.SQLParamExternal):\n buf.write_int32(0) # unspecified\n elif isinstance(param, dbstate.SQLParamExtractedConst):\n buf.write_int32(param.type_oid)\n elif isinstance(param, dbstate.SQLParamGlobal):\n buf.write_int32(0) # unspecified\n assert len(bytes(buf)) == 2 + 4 * len(params)\n return bytes(buf)\n\n\ncdef write_arg(\n buf: WriteBuffer, pg_type: tuple, values: dbstate.SQLSetting\n):\n value = values[0]\n\n if pg_type == ('text',) and isinstance(value, str):\n val = str(value).encode('UTF-8')\n buf.write_len_prefixed_bytes(val)\n elif pg_type == ('uuid',) and isinstance(value, str):\n try:\n id = uuid.UUID(value)\n buf.write_len_prefixed_bytes(id.bytes)\n except ValueError:\n buf.write_int32(-1) # NULL\n elif pg_type == ('int8',) and isinstance(value, int):\n buf.write_int32(8)\n buf.write_int64(value)\n elif pg_type == ('int4',) and isinstance(value, int):\n buf.write_int32(4)\n buf.write_int32(value)\n elif pg_type == ('int2',) and isinstance(value, int):\n buf.write_int32(2)\n buf.write_int16(value)\n elif pg_type == ('bool',):\n is_truthy = is_setting_truthy(value)\n if is_truthy == None:\n buf.write_int32(-1)\n else:\n buf.write_int32(1)\n buf.write_byte(1 if is_truthy else 0)\n elif pg_type == ('float8',) and isinstance(value, float):\n buf.write_int32(8)\n buf.write_double(value)\n elif pg_type == ('float4',) and isinstance(value, float):\n buf.write_int32(4)\n buf.write_float(value)\n else:\n buf.write_int32(-1) # NULL\n raise RuntimeError(\n f\"unimplemented glob type={pg_type}, value={type(value)}\"\n )\n\n\ndef is_setting_truthy(value: str | int | float) -> bool | None:\n if isinstance(value, int):\n return value != 0\n if isinstance(value, str):\n value = value.lower()\n if value == 'o':\n # ambigious\n return None\n\n truthy_values = ('on', 'true', 'yes', '1')\n if any(t.startswith(value) for t in truthy_values):\n return True\n\n falsy_values = ('off', 'false', 'no', '0')\n if any(t.startswith(value) for t in falsy_values):\n return False\n return None\n\n\ncdef inline int16_t read_int16(data: bytes):\n return int.from_bytes(data[0:2], \"big\", signed=True)\n\n\ncdef inline int32_t read_int32(data: bytes):\n return int.from_bytes(data[0:4], \"big\", signed=True)\n\n\ndef new_pg_connection(server, sslctx, endpoint_security, connection_made_at):\n return PgConnection(\n server,\n sslctx,\n endpoint_security,\n passive=False,\n transport=srvargs.ServerConnTransport.TCP_PG,\n external_auth=False,\n connection_made_at=connection_made_at,\n )\n" }, { "path": "edb/server/protocol/protocol.pxd", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2021-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nfrom edb.server.protocol cimport binary\n\n\ncdef class HttpRequest:\n\n cdef:\n public object url\n public bytes version\n public bint should_keep_alive\n public bytes content_type\n public bytes method\n public bytes accept\n public bytes body\n public bytes host\n public bytes origin\n public bytes authorization\n public object params\n public object forwarded\n public object cookies\n\n\ncdef class HttpResponse:\n\n cdef:\n public object status\n public bint close_connection\n public bytes content_type\n public dict custom_headers\n public bytes body\n public bint sent\n\n\ncdef class HttpProtocol:\n\n cdef public object server\n\n cdef:\n object loop\n object parser\n object transport\n object unprocessed\n object sslctx\n object sslctx_pgext\n bint in_response\n bint first_data_call\n bint external_auth\n bint respond_hsts\n bint is_tls\n object binary_endpoint_security\n object http_endpoint_security\n object tenant\n bint is_tenant_host\n object connection_made_at\n\n HttpRequest current_request\n\n cdef _not_found(self, HttpRequest request, HttpResponse response,\n str message = ?)\n cdef _bad_request(self, HttpRequest request, HttpResponse response,\n str message)\n cdef _unauthorized(self, HttpRequest request, HttpResponse response,\n str message)\n cdef _return_binary_error(self, binary.EdgeConnection proto)\n cdef _write(self, bytes req_version, bytes resp_status,\n bytes content_type, dict custom_headers, bytes body,\n bint close_connection)\n\n cpdef write(self, HttpRequest request, HttpResponse response)\n\n cdef unhandled_exception(self, bytes status, ex)\n cdef resume(self)\n cpdef close(self)\n cdef inline _schedule_handle_request(self, request)\n cdef inline _close_with_error(self, bytes status, bytes message)\n" }, { "path": "edb/server/protocol/protocol.pyi", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2019-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\nimport asyncio\nimport http\nimport http.cookies\nimport httptools\nimport ssl\n\nfrom edb.server import args as srvargs\nfrom edb.server import server\n\nclass HttpRequest:\n url: httptools.URL\n version: bytes\n should_keep_alive: bool\n content_type: bytes\n method: bytes\n accept: bytes\n body: bytes\n host: bytes\n origin: bytes\n authorization: bytes\n params: dict[bytes, bytes]\n forwarded: dict[bytes, bytes]\n cookies: http.cookies.SimpleCookie\n\nclass HttpResponse:\n status: http.HTTPStatus\n close_connection: bool\n content_type: bytes\n custom_headers: dict[str, str]\n body: bytes\n sent: bool\n\nclass HttpProtocol(asyncio.Protocol):\n def __init__(\n self,\n server: server.BaseServer,\n sslctx: ssl.SSLContext,\n sslctx_pgext: ssl.SSLContext,\n *,\n external_auth: bool = False,\n binary_endpoint_security: srvargs.ServerEndpointSecurityMode,\n http_endpoint_security: srvargs.ServerEndpointSecurityMode,\n ) -> None:\n ...\n\n def write_raw(self, data: bytes) -> None:\n ...\n\n def write(self, request: HttpRequest, response: HttpResponse) -> None:\n ...\n\n def close(self) -> None:\n ...\n" }, { "path": "edb/server/protocol/protocol.pyx", "content": "\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2021-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\ninclude \"./consts.pxi\"\n\n\nimport asyncio\nimport collections\nimport http\nimport http.cookies\nimport re\nimport ssl\nimport time\nimport urllib.parse\n\nimport httptools\n\nfrom edb import errors\nfrom edb.common import debug\nfrom edb.common import markup\nfrom edb.common.log import current_tenant\n\nfrom edb.graphql import extension as graphql_ext\n\nfrom edb.server import args as srvargs\nfrom edb.server import config, metrics as srv_metrics\nfrom edb.server import tenant as edbtenant\nfrom edb.server.protocol cimport binary\nfrom edb.server.protocol import binary\nfrom edb.server.protocol import pg_ext\nfrom edb.server import defines as edbdef\nfrom edb.server.dbview cimport dbview\n# Without an explicit cimport of `pgproto.debug`, we\n# can't cimport `protocol.binary` for some reason.\nfrom edb.server.pgproto.debug cimport PG_DEBUG\n\nfrom . import auth\nfrom . cimport auth_helpers\nfrom . import edgeql_ext\nfrom . import metrics\nfrom . import server_info\nfrom . import notebook_ext\nfrom . import system_api\nfrom . import ui_ext\nfrom . import auth_ext\nfrom . import ai_ext\n\n\nHTTPStatus = http.HTTPStatus\n\nPROTO_MIME = (\n f'application/x.edgedb.'\n f'v_{edbdef.CURRENT_PROTOCOL[0]}_{edbdef.CURRENT_PROTOCOL[1]}'\n f'.binary'\n).encode()\n\nPROTO_MIME_RE = re.compile(br'application/x\\.edgedb\\.v_(\\d+)_(\\d+)\\.binary')\n\n\ncdef class HttpRequest:\n\n def __cinit__(self):\n self.body = b''\n self.authorization = b''\n self.content_type = b''\n self.forwarded = {}\n self.cookies = http.cookies.SimpleCookie()\n\n\ncdef class HttpResponse:\n\n def __cinit__(self):\n self.status = HTTPStatus.OK\n self.content_type = b'text/plain'\n self.custom_headers = {}\n self.body = b''\n self.close_connection = False\n self.sent = False\n\n\ncdef class HttpProtocol:\n\n def __init__(\n self,\n server,\n sslctx,\n sslctx_pgext,\n *,\n external_auth: bool=False,\n binary_endpoint_security = None,\n http_endpoint_security = None,\n ):\n self.loop = server.get_loop()\n self.server = server\n self.tenant = None\n self.transport = None\n self.external_auth = external_auth\n self.sslctx = sslctx\n self.sslctx_pgext = sslctx_pgext\n\n self.parser = None\n self.current_request = None\n self.in_response = False\n self.unprocessed = None\n self.first_data_call = True\n\n self.binary_endpoint_security = binary_endpoint_security\n self.http_endpoint_security = http_endpoint_security\n self.respond_hsts = False # redirect non-TLS HTTP clients to TLS URL\n\n self.is_tls = False\n self.is_tenant_host = False\n\n def connection_made(self, transport):\n self.connection_made_at = time.monotonic()\n self.transport = transport\n\n def connection_lost(self, exc):\n srv_metrics.client_connection_duration.observe(\n time.monotonic() - self.connection_made_at,\n self.get_tenant_label(),\n \"http\",\n )\n self.transport = None\n self.unprocessed = None\n self.server.maybe_auto_shutdown()\n\n def get_tenant_label(self):\n if self.tenant is None:\n return \"unknown\"\n else:\n return self.tenant.get_instance_name()\n\n def pause_writing(self):\n pass\n\n def resume_writing(self):\n pass\n\n def eof_received(self):\n pass\n\n def data_received(self, data):\n if self.first_data_call:\n self.first_data_call = False\n\n # Detect if the client is speaking TLS in the \"first\" data using\n # the SSL library. This is not the official handshake as we only\n # need to know \"is_tls\"; the first data is used again for the true\n # handshake if is_tls = True. This is for further responding a nice\n # error message to non-TLS clients.\n is_tls = True\n try:\n outgoing = ssl.MemoryBIO()\n incoming = ssl.MemoryBIO()\n incoming.write(data)\n sslobj = self.sslctx.wrap_bio(\n incoming, outgoing, server_side=True\n )\n sslobj.do_handshake()\n except ssl.SSLWantReadError:\n pass\n except ssl.SSLError:\n is_tls = False\n\n self.is_tls = is_tls\n\n if is_tls:\n # Most clients should arrive here to continue with TLS\n self.transport.pause_reading()\n self.loop.create_task(self._forward_first_data(data))\n self.loop.create_task(self._start_tls())\n return\n\n # In case when we're talking to a non-TLS client, keep using the\n # legacy magic byte check to choose the HTTP or binary protocol.\n if data[0:2] == b'V\\x00':\n # This is, most likely, our binary protocol,\n # as its first message kind is `V`.\n #\n # Switch protocols now (for compatibility).\n if (\n self.binary_endpoint_security\n is srvargs.ServerEndpointSecurityMode.Optional\n ):\n self._switch_to_binary_protocol(data)\n else:\n self._return_binary_error(\n self._switch_to_binary_protocol()\n )\n return\n elif data[0:1] == b'\\x00':\n # Postgres protocol, assuming the 1st message is less than 16MB\n pg_ext_conn = pg_ext.new_pg_connection(\n self.server,\n self.sslctx_pgext,\n self.binary_endpoint_security,\n connection_made_at=self.connection_made_at,\n )\n self.transport.set_protocol(pg_ext_conn)\n pg_ext_conn.connection_made(self.transport)\n pg_ext_conn.data_received(data)\n return\n else:\n # HTTP.\n self._init_http_parser()\n self.respond_hsts = (\n self.http_endpoint_security\n is srvargs.ServerEndpointSecurityMode.Tls\n )\n\n try:\n self.parser.feed_data(data)\n except Exception as ex:\n self.unhandled_exception(b'400 Bad Request', ex)\n\n def on_url(self, url: bytes):\n self.current_request.url = httptools.parse_url(url)\n\n def on_header(self, name: bytes, value: bytes):\n name = name.lower()\n if name == b'content-type':\n self.current_request.content_type = value\n elif name == b'host':\n self.current_request.host = value\n elif name == b'origin':\n self.current_request.origin = value\n elif name == b'accept':\n if self.current_request.accept:\n self.current_request.accept += b',' + value\n else:\n self.current_request.accept = value\n elif name == b'authorization':\n self.current_request.authorization = value\n elif name.startswith(b'x-edgedb-'):\n if self.current_request.params is None:\n self.current_request.params = {}\n param = name[len(b'x-edgedb-'):]\n self.current_request.params[param] = value\n elif name.startswith(b'x-gel-'):\n if self.current_request.params is None:\n self.current_request.params = {}\n param = name[len(b'x-gel-'):]\n self.current_request.params[param] = value\n elif name.startswith(b'x-forwarded-'):\n if self.current_request.forwarded is None:\n self.current_request.forwarded = {}\n forwarded_key = name[len(b'x-forwarded-'):]\n self.current_request.forwarded[forwarded_key] = value\n elif name == b'cookie':\n self.current_request.cookies.load(value.decode('ascii'))\n\n def on_body(self, body: bytes):\n self.current_request.body += body\n\n def on_message_begin(self):\n self.current_request = HttpRequest()\n\n def on_message_complete(self):\n self.transport.pause_reading()\n\n req = self.current_request\n\n req.version = self.parser.get_http_version().encode()\n req.should_keep_alive = self.parser.should_keep_alive()\n req.method = self.parser.get_method().upper()\n\n if self.in_response:\n # pipelining support\n if self.unprocessed is None:\n self.unprocessed = collections.deque()\n self.unprocessed.append(req)\n else:\n self.in_response = True\n self._schedule_handle_request(req)\n\n self.server._http_last_minute_requests += 1\n\n cdef inline _schedule_handle_request(self, request):\n if self.tenant is None:\n self.loop.create_task(self._handle_request(request))\n elif self.tenant.is_accepting_connections():\n self.tenant.create_task(\n self._handle_request(request), interruptable=False\n )\n else:\n self._close_with_error(\n b'503 Service Unavailable',\n b'The server is closing.',\n )\n\n cpdef close(self):\n if self.transport is not None:\n self.transport.close()\n self.transport = None\n self.unprocessed = None\n\n cdef unhandled_exception(self, bytes status, ex):\n if debug.flags.server:\n markup.dump(ex)\n\n self._close_with_error(\n status,\n f'{type(ex).__name__}: {ex}'.encode(),\n )\n\n cdef inline _close_with_error(self, bytes status, bytes message):\n self._write(\n b'1.0',\n status,\n b'text/plain',\n {},\n message,\n True)\n\n self.close()\n\n cdef resume(self):\n if self.transport is None:\n return\n\n if self.unprocessed:\n req = self.unprocessed.popleft()\n self._schedule_handle_request(req)\n else:\n self.transport.resume_reading()\n\n cdef _write(self, bytes req_version, bytes resp_status,\n bytes content_type, dict custom_headers, bytes body,\n bint close_connection):\n if self.transport is None:\n return\n data = [\n b'HTTP/', req_version, b' ', resp_status, b'\\r\\n',\n b'Content-Type: ', content_type, b'\\r\\n',\n ]\n if content_type != b\"text/event-stream\":\n data.extend(\n (b'Content-Length: ', f'{len(body)}'.encode(), b'\\r\\n'),\n )\n\n for key, value in custom_headers.items():\n data.append(f'{key}: {value}\\r\\n'.encode())\n\n if close_connection:\n data.append(b'Connection: close\\r\\n')\n data.append(b'\\r\\n')\n if body:\n data.append(body)\n self.transport.write(b''.join(data))\n\n cpdef write(self, HttpRequest request, HttpResponse response):\n assert type(response.status) is HTTPStatus\n self._write(\n request.version,\n f'{response.status.value} {response.status.phrase}'.encode(),\n response.content_type,\n response.custom_headers,\n response.body,\n response.close_connection or not request.should_keep_alive)\n response.sent = True\n\n def write_raw(self, bytes data):\n self.transport.write(data)\n\n def _switch_to_binary_protocol(self, data=None):\n binproto = binary.new_edge_connection(\n self.server,\n self.tenant,\n external_auth=self.external_auth,\n connection_made_at=self.connection_made_at,\n )\n self.transport.set_protocol(binproto)\n binproto.connection_made(self.transport)\n if data:\n binproto.data_received(data)\n return binproto\n\n def _init_http_parser(self):\n self.parser = httptools.HttpRequestParser(self)\n self.current_request = HttpRequest()\n\n async def _forward_first_data(self, data):\n # As we stole the \"first data\", we need to manually send it back to\n # the SSLProtocol. The hack here is highly-coupled with uvloop impl.\n transport = self.transport # The TCP transport\n for i in range(3):\n await asyncio.sleep(0)\n ssl_proto = self.transport.get_protocol()\n if ssl_proto is not self:\n break\n else:\n raise RuntimeError(\"start_tls() hasn't run in 3 loop iterations\")\n\n # Delay for one more iteration to make sure the first data is fed after\n # SSLProtocol.connection_made() is called.\n await asyncio.sleep(0)\n\n data_len = len(data)\n buf = ssl_proto.get_buffer(data_len)\n buf[:data_len] = data\n ssl_proto.buffer_updated(data_len)\n\n async def _start_tls(self):\n self.transport = await self.loop.start_tls(\n self.transport, self, self.sslctx, server_side=True\n )\n sslobj = self.transport.get_extra_info('ssl_object')\n tenant = self.server.retrieve_tenant(sslobj)\n if tenant is edbtenant.host_tenant:\n tenant = None\n self.is_tenant_host = True\n self.tenant = tenant\n if self.tenant is not None:\n current_tenant.set(self.get_tenant_label())\n if sslobj.selected_alpn_protocol() == 'edgedb-binary':\n self._switch_to_binary_protocol()\n else:\n # It's either HTTP as the negotiated protocol, or the negotiation\n # failed and we have no idea what ALPN the client has set. Here we\n # just start talking in HTTP, and let the client bindings decide if\n # this is an error based on the ALPN result.\n self._init_http_parser()\n\n cdef _return_binary_error(self, binary.EdgeConnection proto):\n proto.write_error(errors.BinaryProtocolError(\n 'TLS Required',\n details='The server requires Transport Layer Security (TLS)',\n hint='Upgrade the client to a newer version that supports TLS'\n ))\n proto.close()\n\n async def _handle_request(self, HttpRequest request):\n cdef:\n HttpResponse response = HttpResponse()\n\n if self.transport is None:\n return\n\n if self.respond_hsts:\n if request.host:\n path = request.url.path.lstrip(b'/')\n loc = b'https://' + request.host + b'/' + path\n self.transport.write(\n b'HTTP/1.1 301 Moved Permanently\\r\\n'\n b'Strict-Transport-Security: max-age=31536000\\r\\n'\n b'Location: ' + loc + b'\\r\\n'\n b'\\r\\n'\n )\n else:\n msg = b'Request is missing a header: Host\\r\\n'\n self.transport.write(\n b'HTTP/1.1 400 Bad Request\\r\\n'\n b'Content-Length: ' + str(len(msg)).encode() + b'\\r\\n'\n b'\\r\\n' + msg\n )\n\n self.close()\n return\n\n if self.is_tls:\n if (\n self.http_endpoint_security\n is srvargs.ServerEndpointSecurityMode.Optional\n ):\n response.custom_headers['Strict-Transport-Security'] = \\\n 'max-age=0'\n elif (\n self.http_endpoint_security\n is srvargs.ServerEndpointSecurityMode.Tls\n ):\n response.custom_headers['Strict-Transport-Security'] = \\\n 'max-age=31536000'\n else:\n raise AssertionError(\n f\"unexpected http_endpoint_security \"\n f\"value: {self.http_endpoint_security}\"\n )\n\n try:\n await self.handle_request(request, response)\n except errors.AvailabilityError as ex:\n self._close_with_error(\n b\"503 Service Unavailable\",\n f'{type(ex).__name__}: {ex}'.encode(),\n )\n return\n except Exception as ex:\n self.unhandled_exception(b\"500 Internal Server Error\", ex)\n return\n\n if not response.sent:\n self.write(request, response)\n self.in_response = False\n\n if response.close_connection or not request.should_keep_alive:\n self.close()\n else:\n self.resume()\n\n def check_readiness(self):\n if self.tenant.is_blocked():\n readiness_reason = self.tenant.get_readiness_reason()\n msg = \"the server is not accepting requests\"\n if readiness_reason:\n msg = f\"{msg}: {readiness_reason}\"\n raise errors.ServerBlockedError(msg)\n elif not self.tenant.is_online():\n readiness_reason = self.tenant.get_readiness_reason()\n msg = \"the server is going offline\"\n if readiness_reason:\n msg = f\"{msg}: {readiness_reason}\"\n raise errors.ServerOfflineError(msg)\n\n async def handle_request(self, HttpRequest request, HttpResponse response):\n request_url = get_request_url(request, self.is_tls)\n path = request_url.path.decode('ascii')\n path = path.strip('/')\n path_parts = path.split('/')\n path_parts_len = len(path_parts)\n route = path_parts[0]\n\n if self.tenant is None and route in ['db', 'auth', 'branch']:\n self.tenant = self.server.get_default_tenant()\n self.check_readiness()\n if self.tenant.is_accepting_connections():\n return await self.tenant.create_task(\n self.handle_request(request, response),\n interruptable=False,\n )\n else:\n return self._close_with_error(\n b'503 Service Unavailable',\n b'The server is closing.',\n )\n\n if route in ['db', 'branch']:\n if path_parts_len < 2:\n return self._not_found(request, response)\n\n dbname = urllib.parse.unquote(path_parts[1])\n dbname = self.tenant.resolve_branch_name(\n database=dbname if route == 'db' else None,\n branch=dbname if route == 'branch' else None,\n )\n extname = path_parts[2] if path_parts_len > 2 else None\n\n # Binary proto tunnelled through HTTP\n if extname is None:\n if await self._handle_cors(\n request, response,\n dbname=dbname,\n allow_methods=['POST'],\n allow_headers=[\n 'Authorization', 'X-EdgeDB-User', 'X-Gel-User'\n ],\n ):\n return\n\n if request.method == b'POST':\n if not request.content_type:\n return self._bad_request(\n request,\n response,\n message=\"missing or malformed Content-Type header\",\n )\n\n ver_m = PROTO_MIME_RE.match(request.content_type)\n if not ver_m:\n return self._bad_request(\n request,\n response,\n message=\"missing or malformed Content-Type header\",\n )\n\n proto_ver = (\n int(ver_m.group(1).decode()),\n int(ver_m.group(2).decode()),\n )\n\n if proto_ver < edbdef.MIN_PROTOCOL:\n return self._bad_request(\n request,\n response,\n message=\"requested protocol version is too old and \"\n \"no longer supported\",\n )\n if proto_ver > edbdef.CURRENT_PROTOCOL:\n proto_ver = edbdef.CURRENT_PROTOCOL\n\n params = request.params\n if params is None:\n conn_params = {}\n else:\n conn_params = {\n n.decode(\"utf-8\"): v.decode(\"utf-8\")\n for n, v in request.params.items()\n }\n\n conn_params[\"database\"] = dbname\n\n response.body = await binary.eval_buffer(\n self.server,\n self.tenant,\n database=dbname,\n data=self.current_request.body,\n conn_params=conn_params,\n protocol_version=proto_ver,\n auth_data=self.current_request.authorization,\n transport=srvargs.ServerConnTransport.HTTP,\n tcp_transport=self.transport,\n )\n response.status = http.HTTPStatus.OK\n response.content_type = (\n f'application/x.edgedb.v_'\n f'{proto_ver[0]}_{proto_ver[1]}.binary'\n ).encode()\n response.close_connection = True\n\n else:\n if await self._handle_cors(\n request, response,\n dbname=dbname,\n allow_methods=['GET', 'POST'],\n allow_headers=[\n 'Authorization', 'X-EdgeDB-User', 'X-Gel-User'\n ],\n expose_headers=(\n ['EdgeDB-Protocol-Version', 'Gel-Protocol-Version']\n if extname == 'notebook'\n else ['WWW-Authenticate'] if extname != 'auth'\n else None\n ),\n allow_credentials=True\n ):\n return\n\n # Check if this is a request to a registered extension\n if extname == 'edgeql':\n extname = 'edgeql_http'\n if extname == 'ext':\n if path_parts_len < 4:\n return self._not_found(request, response)\n extname = path_parts[3]\n args = path_parts[4:]\n else:\n args = path_parts[3:]\n\n role_name = None\n if extname != 'auth':\n role_name = await self._check_http_auth(\n request, response, dbname\n )\n if not role_name:\n return\n\n db = self.tenant.maybe_get_db(dbname=dbname)\n if db is None:\n return self._not_found(request, response)\n\n if extname not in db.extensions:\n return self._not_found(request, response)\n\n if extname == 'graphql':\n await graphql_ext.handle_request(\n request, response, db, role_name, args, self.tenant\n )\n elif extname == 'notebook':\n await notebook_ext.handle_request(\n request, response, db, role_name, args, self.tenant\n )\n elif extname == 'edgeql_http':\n await edgeql_ext.handle_request(\n request, response, db, role_name, args, self.tenant\n )\n elif extname == 'ai':\n await ai_ext.handle_request(\n self,\n request, response, db, role_name, args, self.tenant\n )\n elif extname == 'auth':\n netloc = (\n f\"{request_url.host.decode()}:{request_url.port}\"\n if request_url.port\n else request_url.host.decode()\n )\n ext_base_path = f\"{request_url.schema.decode()}://\" \\\n f\"{netloc}/{route}/\" \\\n f\"{urllib.parse.quote(dbname)}/ext/auth\"\n handler = auth_ext.http.Router(\n db=db,\n base_path=ext_base_path,\n tenant=self.tenant,\n )\n await handler.handle_request(request, response, args)\n if args:\n if args[0] == 'ui':\n if not (len(args) > 1 and args[1] == \"_static\"):\n srv_metrics.auth_ui_renders.inc(\n 1.0, self.get_tenant_label()\n )\n else:\n srv_metrics.auth_api_calls.inc(\n 1.0, self.get_tenant_label()\n )\n else:\n return self._not_found(request, response)\n\n elif route == 'auth':\n if await self._handle_cors(\n request, response,\n allow_methods=['GET'],\n allow_headers=['Authorization'],\n expose_headers=['WWW-Authenticate', 'Authentication-Info']\n ):\n return\n\n # Authentication request\n await auth.handle_request(\n request,\n response,\n path_parts[1:],\n self.tenant,\n )\n elif route == 'server':\n if not await self._authenticate_for_default_conn_transport(\n request,\n response,\n srvargs.ServerConnTransport.HTTP_HEALTH,\n ):\n return\n\n # System API request\n await system_api.handle_request(\n request,\n response,\n path_parts[1:],\n self.server,\n self.tenant,\n is_tenant_host=self.is_tenant_host,\n )\n elif path_parts == ['metrics'] and request.method == b'GET':\n if not await self._authenticate_for_default_conn_transport(\n request,\n response,\n srvargs.ServerConnTransport.HTTP_METRICS,\n ):\n return\n\n self.server.get_compiler_pool().refresh_metrics()\n # Quoting the Open Metrics spec:\n # Implementers MUST expose metrics in the OpenMetrics\n # text format in response to a simple HTTP GET request\n # to a documented URL for a given process or device.\n # This endpoint SHOULD be called \"/metrics\".\n await metrics.handle_request(\n request,\n response,\n self.tenant,\n )\n elif (path_parts == ['server-info'] and\n request.method == b'GET' and\n (self.server.in_dev_mode() or self.server.in_test_mode())\n ):\n await server_info.handle_request(\n request,\n response,\n self.server,\n )\n elif path_parts[0] == 'ui':\n if not self.server.is_admin_ui_enabled():\n return self._not_found(\n request,\n response,\n \"Admin UI is not enabled on this EdgeDB instance. \"\n \"Run the server with --admin-ui=enabled \"\n \"(or EDGEDB_SERVER_ADMIN_UI=enabled) to enable.\"\n )\n else:\n await ui_ext.handle_request(\n request,\n response,\n path_parts[1:],\n self.server,\n )\n else:\n return self._not_found(request, response)\n\n cdef _not_found(\n self,\n HttpRequest request,\n HttpResponse response,\n str message = \"Unknown path\",\n ):\n response.body = message.encode(\"utf-8\")\n response.status = http.HTTPStatus.NOT_FOUND\n response.close_connection = True\n\n cdef _bad_request(\n self,\n HttpRequest request,\n HttpResponse response,\n str message,\n ):\n response.body = message.encode(\"utf-8\")\n response.status = http.HTTPStatus.BAD_REQUEST\n response.close_connection = True\n\n async def _handle_cors(\n self,\n HttpRequest request,\n HttpResponse response,\n *,\n str dbname = None,\n list allow_methods = None,\n list allow_headers = [],\n list expose_headers = None,\n bint allow_credentials = False\n ):\n db = self.tenant.maybe_get_db(dbname=dbname) if dbname else None\n\n config = None\n if db is not None:\n if db.db_config is None:\n await db.introspection()\n\n config = db.db_config.get('cors_allow_origins')\n if config is None:\n config = self.tenant.get_sys_config().get('cors_allow_origins')\n\n allowed_origins = config.value if config else None\n overrides = self.server.get_cors_always_allowed_origins()\n\n if allowed_origins is None and overrides == []:\n return False\n\n origin = request.origin.decode() if request.origin else None\n\n origin_allowed = origin is not None and (\n any(\n override.match(origin) if isinstance(override, re.Pattern)\n else origin == override\n for override in overrides\n )\n or (origin in allowed_origins or '*' in allowed_origins)\n )\n\n if origin_allowed:\n response.custom_headers['Access-Control-Allow-Origin'] = origin\n if expose_headers is not None:\n response.custom_headers['Access-Control-Expose-Headers'] = (\n ', '.join(expose_headers))\n\n if request.method == b'OPTIONS':\n response.status = http.HTTPStatus.NO_CONTENT\n if origin_allowed:\n if allow_methods is not None:\n response.custom_headers['Access-Control-Allow-Methods'] = (\n ', '.join(allow_methods))\n response.custom_headers['Access-Control-Allow-Headers'] = (\n ', '.join(['Content-Type'] + allow_headers))\n if allow_credentials:\n response.custom_headers['Access-Control-Allow-Credentials'] = (\n 'true')\n\n return True\n\n return False\n\n cdef _unauthorized(\n self,\n HttpRequest request,\n HttpResponse response,\n str message,\n ):\n response.body = message.encode(\"utf-8\")\n response.status = http.HTTPStatus.UNAUTHORIZED\n response.close_connection = True\n\n async def _check_http_auth(\n self,\n HttpRequest request,\n HttpResponse response,\n str dbname,\n ):\n dbindex: dbview.DatabaseIndex = self.tenant._dbindex\n\n scheme = None\n try:\n # Extract the username from the relevant request headers\n scheme, auth_payload = auth_helpers.extract_token_from_auth_data(\n request.authorization)\n username, opt_password = auth_helpers.extract_http_user(\n scheme, auth_payload, request.params)\n username = self.tenant.resolve_user_name(username)\n\n # Fetch the configured auth methods\n authmethods = await self.tenant.get_auth_methods(\n username, srvargs.ServerConnTransport.SIMPLE_HTTP)\n\n auth_errors = {}\n\n for authmethod in authmethods:\n authmethod_name = authmethod._tspec.name.split('::')[1]\n try:\n # If the auth method and the provided auth information\n # match, try to resolve the authentication.\n if authmethod_name == 'JWT' and scheme == 'bearer':\n auth_helpers.auth_jwt(\n self.tenant, auth_payload, username, dbname)\n elif authmethod_name == 'Password' and scheme == 'basic':\n auth_helpers.auth_basic(\n self.tenant, username, opt_password)\n elif authmethod_name == 'Trust':\n pass\n elif authmethod_name == 'SCRAM':\n raise errors.AuthenticationError(\n 'authentication failed: '\n 'SCRAM authentication required but not '\n 'supported for HTTP'\n )\n elif authmethod_name == 'mTLS':\n if (\n self.http_endpoint_security\n is srvargs.ServerEndpointSecurityMode.Tls\n or self.is_tls\n ):\n auth_helpers.auth_mtls_with_user(\n self.transport, username)\n else:\n raise errors.AuthenticationError(\n 'authentication failed: wrong method used')\n\n except errors.AuthenticationError as e:\n auth_errors[authmethod_name] = e\n\n else:\n break\n\n if len(auth_errors) == len(authmethods):\n if len(auth_errors) > 1:\n desc = \"; \".join(\n f\"{k}: {e.args[0]}\" for k, e in auth_errors.items())\n raise errors.AuthenticationError(\n f\"all authentication methods failed: {desc}\")\n else:\n raise next(iter(auth_errors.values()))\n\n role = self.tenant.get_roles().get(username)\n if not role:\n raise errors.AuthenticationError('authentication failed')\n branches = role['branches']\n if '*' not in branches and dbname not in branches:\n raise errors.AuthenticationError(\n f\"authentication failed: user does not have permission for \"\n f\"database branch '{dbname}'\"\n )\n\n except Exception as ex:\n if debug.flags.server:\n markup.dump(ex)\n\n self._unauthorized(request, response, str(ex))\n\n # If no scheme was specified, add a WWW-Authenticate header\n if scheme == '':\n response.custom_headers['WWW-Authenticate'] = (\n 'Basic realm=\"edgedb\", Bearer'\n )\n\n return None\n\n return username\n\n async def _authenticate_for_default_conn_transport(\n self,\n HttpRequest request,\n HttpResponse response,\n transport: srvargs.ServerConnTransport,\n ):\n try:\n auth_methods = self.server.get_default_auth_methods(transport)\n auth_errors = {}\n\n for auth_method in auth_methods:\n authmethod_name = auth_method._tspec.name.split('::')[1]\n try:\n # If the auth method and the provided auth information\n # match, try to resolve the authentication.\n if authmethod_name == 'Trust':\n pass\n elif authmethod_name == 'mTLS':\n if (\n self.http_endpoint_security\n is srvargs.ServerEndpointSecurityMode.Tls\n or self.is_tls\n ):\n auth_helpers.auth_mtls(self.transport)\n else:\n raise errors.AuthenticationError(\n 'authentication failed: wrong method used')\n except errors.AuthenticationError as e:\n auth_errors[authmethod_name] = e\n else:\n break\n\n if len(auth_errors) == len(auth_methods):\n if len(auth_errors) > 1:\n desc = \"; \".join(\n f\"{k}: {e.args[0]}\" for k, e in auth_errors.items())\n raise errors.AuthenticationError(\n f\"all authentication methods failed: {desc}\")\n else:\n raise next(iter(auth_errors.values()))\n\n except Exception as ex:\n if debug.flags.server:\n markup.dump(ex)\n\n self._unauthorized(request, response, str(ex))\n\n return False\n\n return True\n\ndef get_request_url(request, is_tls):\n request_url = request.url\n default_schema = b\"https\" if is_tls else b\"http\"\n if all(\n getattr(request_url, attr) is None\n for attr in ('schema', 'host', 'port')\n ):\n forwarded = request.forwarded if hasattr(request, 'forwarded') else {}\n schema = forwarded.get(b'proto', default_schema).decode()\n host_header = forwarded.get(b'host', request.host).decode()\n\n host, _, port = host_header.partition(':')\n path = request_url.path.decode()\n new_url = f\"{schema}://\"\\\n f\"{host}{port and ':' + port}\"\\\n f\"{path}\"\n request_url = httptools.parse_url(new_url.encode())\n\n return request_url\n" }, { "path": "edb/server/protocol/request_scheduler.py", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\nfrom __future__ import annotations\nfrom dataclasses import dataclass, field\nfrom typing import (\n Final,\n Iterable,\n Literal,\n Optional,\n Sequence,\n)\n\nimport abc\nimport asyncio\nimport copy\nimport random\n\n\n@dataclass\nclass Timer:\n \"\"\"Represents a time after which an action should be taken.\n\n Examples:\n (None, True) = execute immediately\n (None, False) = execute any time\n (10, True) = execute immediately after 10s\n (10, False) = execute any time after 10s\n \"\"\"\n\n time: Optional[float] = None\n\n # Whether the action should be taken as soon as possible after the time.\n urgent: bool = True\n\n @staticmethod\n def create_delay(delay: Optional[float], urgent: bool) -> Timer:\n now = asyncio.get_running_loop().time()\n if delay is None:\n time = None\n else:\n now = asyncio.get_running_loop().time()\n time = now + delay\n\n return Timer(time, urgent)\n\n def is_ready(self) -> bool:\n now = asyncio.get_running_loop().time()\n return self.time is None or self.time <= now\n\n def is_ready_and_urgent(self) -> bool:\n return self.is_ready() and self.urgent\n\n def remaining_time(self, max_delay: float) -> float:\n \"\"\"How long before this timer is ready in seconds.\"\"\"\n if self.urgent:\n if self.time is None:\n return 0\n\n else:\n # 1ms extra, just in case\n now = asyncio.get_running_loop().time()\n delay = self.time - now + 0.001\n return min(max(0, delay), max_delay)\n\n else:\n # If not urgent, wait as long as possible\n return max_delay\n\n @staticmethod\n def combine(timers: Iterable[Timer]) -> Optional[Timer]:\n \"\"\"Combine the timers to determine the when to take the next action.\n\n If the timers are (1, False), (2, False), (3, True), it may be wasteful\n to act at times [1, 2, 3].\n\n Instead, we would prefer to act only once, at time 3, since only the\n third action was urgent.\n \"\"\"\n for target_urgency in [True, False]:\n if any(\n timer.time is None and timer.urgent == target_urgency\n for timer in timers\n ):\n # An action should be taken right away.\n return Timer(None, target_urgency)\n\n urgent_times = [\n timer.time\n for timer in timers\n if timer.time is not None and timer.urgent == target_urgency\n ]\n if len(urgent_times) > 0:\n # An action should be taken after some delay\n return Timer(min(urgent_times), target_urgency)\n\n # Nothing to do\n return None\n\n\ndef _default_delay_time() -> Timer:\n return Timer()\n\n\n@dataclass\nclass Scheduler[_T](abc.ABC):\n \"\"\"A scheduler for requests to an asynchronous service.\n\n A Scheduler both generates requests and tracks when the service can be\n accessed.\n \"\"\"\n\n service: Service\n\n # The next time to process requests\n timer: Timer = field(default_factory=_default_delay_time)\n\n @abc.abstractmethod\n async def get_params(\n self, context: Context,\n ) -> Optional[Sequence[Params[_T]]]:\n \"\"\"Get parameters for the requests to run.\"\"\"\n raise NotImplementedError\n\n async def process(self, context: Context) -> bool:\n if not self.timer.is_ready():\n return False\n\n request_params: Optional[Sequence[Params[_T]]]\n try:\n request_params = await self.get_params(context)\n except Exception:\n request_params = None\n\n error_count = 0\n deferred_costs: dict[str, int] = {\n limit_name: 0\n for limit_name in self.service.limits\n }\n success_count = 0\n\n if request_params is None:\n error_count = 1\n\n elif len(request_params) > 0:\n try:\n execution_report = await execute_no_sleep(\n request_params, service=self.service,\n )\n\n except Exception:\n execution_report = ExecutionReport(unknown_error_count=1)\n\n assert isinstance(execution_report, ExecutionReport)\n\n self.finalize(execution_report)\n\n # Cache limits for next time\n if execution_report.updated_limits is not None:\n for limit_name, service_limit in self.service.limits.items():\n if limit_name not in execution_report.updated_limits:\n continue\n\n updated_limit = execution_report.updated_limits[limit_name]\n\n if service_limit is not None:\n service_limit.update_total(updated_limit)\n service_limit.delay_factor = updated_limit.delay_factor\n\n else:\n self.service.limits[limit_name] = updated_limit\n\n # Update counts\n error_count = (\n len(execution_report.known_error_messages)\n + execution_report.unknown_error_count\n )\n\n deferred_costs = execution_report.deferred_costs\n success_count = execution_report.success_count\n\n # Update when this service should be processed again\n self.timer = self.service.next_delay(\n success_count, deferred_costs, error_count, context.naptime\n )\n\n return True\n\n @abc.abstractmethod\n def finalize(self, execution_report: ExecutionReport) -> None:\n \"\"\"An optional final step after executing requests\"\"\"\n pass\n\n\n@dataclass\nclass Context:\n \"\"\"Information passed to a Scheduler to process requests.\"\"\"\n\n # If there is no work, the scheduler should take a nap.\n naptime: float\n\n\n@dataclass\nclass ExecutionReport:\n \"\"\"Information about the requests after they are complete\"\"\"\n\n success_count: int = 0\n unknown_error_count: int = 0\n known_error_messages: list[str] = field(default_factory=list)\n deferred_costs: dict[str, int] = field(default_factory=dict)\n\n # Some requests may report an update to the service's rate limits.\n updated_limits: dict[str, Limits] = field(default_factory=dict)\n\n\n@dataclass\nclass Service:\n \"\"\"Information on how to access to a given service.\"\"\"\n\n # Information about the service's rate limits\n # Even if no Limit is available, the presence of a key indicates that a\n # limit is used at least sometimes.\n limits: dict[str, Optional[Limits]] = field(default_factory=dict)\n\n # The maximum number of times to retry requests\n max_retry_count: Final[int] = 4\n # Whether to jitter the delay time if a retry error is produced\n jitter: Final[bool] = True\n # Initial guess for the delay\n guess_delay: Final[float] = 1.0\n # The upper bound for delays\n delay_max: Final[float] = 60.0\n\n def next_delay(\n self,\n success_count: int,\n deferred_costs: dict[str, int],\n error_count: int,\n naptime: float\n ) -> Timer:\n \"\"\"When should the service should be processed again.\"\"\"\n\n if self.limits is not None:\n # Find the limit with the largest delay\n limit_delays: dict[str, Optional[float]] = {}\n for limit_names, service_limit in self.limits.items():\n if service_limit is None:\n # If no information is available, assume no limits\n limit_delays[limit_names] = None\n\n else:\n base_delay = service_limit.base_delay(\n deferred_costs[limit_names],\n guess=self.guess_delay,\n )\n if base_delay is None:\n limit_delays[limit_names] = None\n\n else:\n # If delay_factor is very high, it may take quite a long\n # time for it to return to 1. A maximum delay prevents\n # this service from never getting checked.\n limit_delays[limit_names] = (\n min(\n base_delay * service_limit.delay_factor,\n self.delay_max,\n )\n )\n delay = _get_maximum_delay(limit_delays)\n\n else:\n # We have absolutely no information about the delay, assume naptime.\n delay = naptime\n\n if error_count > 0:\n # There was an error, wait before trying again.\n # Use the larger of delay or naptime.\n delay = max(delay, naptime) if delay is not None else naptime\n urgent = False\n\n elif any(\n deferred_cost > 0\n for deferred_cost in deferred_costs.values()\n ):\n # There is some deferred work, apply the delay and run immediately.\n urgent = True\n\n elif success_count > 0:\n # Some work was done successfully. Run again to ensure no more work\n # needs to be done.\n delay = None\n urgent = True\n\n else:\n # No work left to do, wait before trying again.\n # Use the larger of delay or naptime.\n delay = max(delay, naptime) if delay is not None else naptime\n urgent = False\n\n return Timer.create_delay(delay, urgent)\n\n\n@dataclass\nclass Limits:\n \"\"\"Information about a service's rate limits.\"\"\"\n\n # Total limit of a resource per minute for a service.\n total: Optional[int | Literal['unlimited']] = None\n\n # Remaining resources before the limit is hit.\n # It is assumed to be decreasing during a call to execute_no_sleep.\n #\n # This can be set by users before a call to Scheduler.process.\n # It will also be updated during execution if a responseincludes an updated\n # value.\n #\n # Finally, it is reset after requests are executed since we don't know when\n # the next call will be.\n remaining: Optional[int] = None\n\n # A delay factor to implement exponential backoff\n delay_factor: float = 1\n\n def base_delay(\n self,\n request_cost: int,\n *,\n guess: float,\n ) -> Optional[float]:\n if self.total == 'unlimited':\n return None\n\n if self.remaining is not None and request_cost <= self.remaining:\n return None\n\n if self.total is not None:\n assert isinstance(self.total, int)\n return 60.0 / self.total * 1.1 # 10% buffer just in case\n\n # guess the delay\n return guess\n\n def update_total(self, latest: Limits) -> Limits:\n \"\"\"Update total based on the latest information.\n\n The total will change rarely. Always take the latest value if\n it exists\n \"\"\"\n if latest.total is not None:\n self.total = latest.total\n\n return self\n\n def update_remaining(self, latest: Limits) -> Limits:\n \"\"\"Update remaining based on the latest information.\n\n The remaining amount is assumed to decreasing during a call to\n execute_no_sleep.\n \"\"\"\n if self.remaining is None:\n self.remaining = latest.remaining\n elif latest.remaining is not None:\n self.remaining = min(self.remaining, latest.remaining)\n\n if self.total == 'unlimited' and self.remaining:\n # If there is a remaining value, the total is not actually\n # unlimited.\n self.total = None\n\n return self\n\n\nclass Request[_T](abc.ABC):\n \"\"\"Represents an async request\"\"\"\n\n params: Params[_T]\n _inner: asyncio.Task[Optional[Result[_T]]]\n\n def __init__(self, params: Params[_T]):\n self.params = params\n self._inner = asyncio.create_task(self.run())\n\n @abc.abstractmethod\n async def run(self) -> Optional[Result[_T]]:\n \"\"\"Run the request and return a result.\"\"\"\n raise NotImplementedError\n\n async def wait_result(self) -> None:\n \"\"\"Wait for the request to complete.\"\"\"\n await self._inner\n\n def get_result(self) -> Optional[Result[_T]]:\n \"\"\"Get the result of the request.\"\"\"\n result = self._inner.result()\n return result\n\n\nclass Params[_T](abc.ABC):\n \"\"\"The parameters of an async request.\n\n These are used to generate requests.\n\n A single Params instance may be used to generate multiple Request\n instances if it fails, but can be retried right away.\n \"\"\"\n\n @abc.abstractmethod\n def costs(self) -> dict[str, int]:\n \"\"\"Expected cost to execute the request.\n\n Keys must match service rate limits.\"\"\"\n raise NotImplementedError\n\n @abc.abstractmethod\n def create_request(self) -> Request[_T]:\n \"\"\"Create a request with the parameters.\"\"\"\n raise NotImplementedError\n\n\n@dataclass(frozen=True)\nclass Result[_T](abc.ABC):\n \"\"\"The result of an async request.\"\"\"\n\n data: _T | Error\n\n # Some services can return request limits along with their usual results.\n # Keys should be a subset of service limits.\n limits: dict[str, Limits] = field(default_factory=dict)\n\n @abc.abstractmethod\n async def finalize(self) -> None:\n \"\"\"An optional finalize to be run sequentially.\"\"\"\n pass\n\n\n@dataclass(frozen=True)\nclass Error:\n \"\"\"Represents an error from an async request.\"\"\"\n\n message: str\n\n # If there was an error, it may be possible to retry the request\n # Eg. 429 too many requests\n retry: bool\n\n\nasync def execute_no_sleep[_T](\n params: Sequence[Params[_T]],\n *,\n service: Service,\n) -> ExecutionReport:\n \"\"\"Attempt to execute as many requests as possible without sleeping.\"\"\"\n report = ExecutionReport()\n\n # Set up limits\n execute_limits: dict[str, Limits] = {\n limit_name: (\n # If no other information is available, for the first attempt assume\n # there is no limit.\n Limits(total='unlimited')\n if service_limit is None else\n copy.copy(service_limit)\n )\n for limit_name, service_limit in service.limits.items()\n }\n\n # If any requests fail and can be retried, retry them up to a maximum number\n # of times.\n retry_count: int = 0\n\n # If the costs are larger than a total limit, set aside the excess to be\n # processed later.\n # This prevents wasting resources, and allows the delays to increase\n # specifically when an unexpected deferral happens.\n pending_request_indexes: list[int]\n excess_request_indexes: list[int]\n\n initial_pending_cost = {\n limit_name: 0\n for limit_name in service.limits.keys()\n }\n for request_index in range(len(params)):\n for limit_name, cost in params[request_index].costs().items():\n initial_pending_cost[limit_name] += cost\n\n if (\n # If the pending cost exceeds a known limit, set aside some\n # requests.\n any(\n limit.total < initial_pending_cost[limit_name]\n for limit_name, limit in service.limits.items()\n if limit is not None and isinstance(limit.total, int)\n )\n\n # Always include at least 1 request\n and request_index != 0\n ):\n pending_request_indexes = (\n list(range(request_index))\n )\n excess_request_indexes = (\n list(range(request_index, len(params)))\n )\n break\n\n else:\n # All inputs can be processed\n pending_request_indexes = list(range(len(params)))\n excess_request_indexes = []\n\n while pending_request_indexes and retry_count < service.max_retry_count:\n # Find the highest delay required by any of the service's limits\n limit_base_delays = _get_limit_base_delays(\n params, execute_limits, pending_request_indexes, service.guess_delay\n )\n base_delay = _get_maximum_delay(limit_base_delays)\n\n active_request_indexes: list[int]\n inactive_request_indexes: list[int]\n if base_delay is None:\n # Try to execute all requests.\n active_request_indexes = pending_request_indexes\n inactive_request_indexes = []\n\n elif retry_count == 0:\n # If there is any delay, only execute one request.\n # This may update the remaining limit, allowing the remaining\n # requests to run.\n active_request_indexes = pending_request_indexes[:1]\n inactive_request_indexes = pending_request_indexes[1:]\n\n else:\n break\n\n results = await _execute_specified(\n params, active_request_indexes,\n )\n\n # Check results\n retry_request_indexes: list[int] = []\n\n for request_index in active_request_indexes:\n if request_index not in results:\n report.unknown_error_count += 1\n continue\n\n result = results[request_index]\n\n if isinstance(result.data, Error):\n if result.data.retry:\n # requests can be retried\n retry_request_indexes.append(request_index)\n\n else:\n # error with message\n report.known_error_messages.append(result.data.message)\n else:\n report.success_count += 1\n\n await result.finalize()\n\n if result.limits is not None:\n for limit_name, execute_limit in execute_limits.items():\n if limit_name not in result.limits:\n continue\n result_limit = result.limits[limit_name]\n execute_limit.update_total(result_limit)\n execute_limit.update_remaining(result_limit)\n\n retry_count += 1\n pending_request_indexes = (\n retry_request_indexes + inactive_request_indexes\n )\n\n # Determine which limits cause unexpected deferrals and require additional\n # delays.\n limit_base_delays = _get_limit_base_delays(\n params, execute_limits, pending_request_indexes, service.guess_delay\n )\n expected_pending_cost = {\n limit_name\n for limit_name in service.limits.keys()\n if limit_base_delays[limit_name] is not None\n }\n if len(expected_pending_cost) == 0:\n # If requests were deferred, but no limit appears to be the cause, delay\n # them all just in case.\n expected_pending_cost = set(service.limits.keys())\n\n # Update deferred costs and any resulting limits.\n report.deferred_costs = {\n limit_name: 0\n for limit_name in service.limits\n }\n for limit_name in service.limits.keys():\n unexpected_deferred_costs = sum(\n params[i].costs()[limit_name]\n for i in pending_request_indexes\n )\n excess_deferred_costs = sum(\n params[i].costs()[limit_name]\n for i in excess_request_indexes\n )\n report.deferred_costs[limit_name] = (\n unexpected_deferred_costs + excess_deferred_costs\n )\n\n if (\n unexpected_deferred_costs != 0\n\n # If the limit was not a cause of delays, don't increase the delay\n # factor.\n and limit_name in expected_pending_cost\n ):\n # If there are deferred requests, gradually increase the delay\n # factor\n execute_limits[limit_name].delay_factor *= (\n 1 + random.random() if service.jitter else 2\n )\n\n elif (\n len(report.known_error_messages) == 0\n and report.unknown_error_count == 0\n and excess_deferred_costs == 0\n ):\n # If there are no errors, gradually decrease the delay factor over\n # time.\n execute_limits[limit_name].delay_factor = max(\n 0.95 * execute_limits[limit_name].delay_factor,\n 1,\n )\n\n # We don't know when the service will be called again, so just clear the\n # remaining values\n for execute_limit in execute_limits.values():\n execute_limit.remaining = None\n\n # Return the updated request limits\n report.updated_limits = execute_limits\n\n return report\n\n\nasync def _execute_specified[_T](\n params: Sequence[Params[_T]],\n indexes: Iterable[int],\n) -> dict[int, Result[_T]]:\n # Send all requests at once.\n # We are confident that all requests can be handled right away.\n\n requests: dict[int, Request[_T]] = {}\n\n for request_index in indexes:\n requests[request_index] = params[request_index].create_request()\n\n results: dict[int, Result[_T]] = {}\n\n for request_index, request in requests.items():\n await request.wait_result()\n\n result = request.get_result()\n if result is not None:\n results[request_index] = result\n\n return results\n\n\ndef _get_limit_base_delays[_T](\n params: Sequence[Params[_T]],\n limits: dict[str, Limits],\n request_indexes: Sequence[int],\n guess_delay: float,\n) -> dict[str, Optional[float]]:\n base_delays = {}\n for limit_name, limit in limits.items():\n pending_limit_cost = sum(\n params[request_index].costs()[limit_name]\n for request_index in request_indexes\n )\n\n base_delays[limit_name] = (limit.base_delay(\n pending_limit_cost, guess=guess_delay,\n ))\n\n return base_delays\n\n\ndef _get_maximum_delay(\n delays: dict[str, Optional[float]]\n) -> Optional[float]:\n result: Optional[float] = None\n for delay in delays.values():\n if result is None:\n result = delay\n elif delay is not None:\n result = max(result, delay)\n\n return result\n" }, { "path": "edb/server/protocol/server_info.py", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2021-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nfrom __future__ import annotations\nfrom typing import Any, TYPE_CHECKING\nimport dataclasses\nimport http\nimport json\n\nimport immutables\n\nfrom edb import errors\n\nfrom edb.ir import statypes\n\nfrom edb.common import debug\nfrom edb.common import markup\n\nif TYPE_CHECKING:\n from edb.server import server as edbserver\n from edb.server.protocol import protocol\n\n\nclass ImmutableEncoder(json.JSONEncoder):\n\n def default(self, obj: Any) -> Any:\n if isinstance(obj, (set, frozenset)):\n return list(obj)\n if isinstance(obj, immutables.Map):\n return dict(obj.items())\n if dataclasses.is_dataclass(obj) and not isinstance(obj, type):\n return dataclasses.asdict(obj)\n if isinstance(obj, statypes.ScalarType):\n return obj.to_json()\n if isinstance(obj, statypes.CompositeType):\n return obj.to_json_value()\n return super().default(obj)\n\n\nasync def handle_request(\n request: protocol.HttpRequest,\n response: protocol.HttpResponse,\n server: edbserver.Server,\n) -> None:\n try:\n output = ImmutableEncoder().encode(server.get_debug_info())\n response.status = http.HTTPStatus.OK\n response.content_type = b'application/json'\n response.body = output.encode()\n response.close_connection = True\n\n except Exception as ex:\n if debug.flags.server:\n markup.dump(ex)\n\n # XXX Fix this when LSP \"location\" objects are implemented\n ex_type = errors.InternalServerError\n\n _response_error(\n response, http.HTTPStatus.INTERNAL_SERVER_ERROR, str(ex), ex_type\n )\n\n\ndef _response_error(\n response: protocol.HttpResponse,\n status: http.HTTPStatus,\n message: str,\n ex_type: type[errors.EdgeDBError],\n) -> None:\n response.body = (\n f'Unexpected error in /server-info.\\n\\n'\n f'{ex_type.__name__}: {message}'\n ).encode()\n response.status = status\n response.close_connection = True\n" }, { "path": "edb/server/protocol/system_api.py", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2019-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\nfrom __future__ import annotations\nfrom typing import TYPE_CHECKING\n\nimport asyncio\nimport http\nimport json\n\nfrom edb import errors\n\nfrom edb.common import debug\nfrom edb.common import markup\n\n\nif TYPE_CHECKING:\n from edb.server import tenant as edbtenant, server as edbserver\n from edb.server.protocol import protocol\n\n\nasync def handle_request(\n request: protocol.HttpRequest,\n response: protocol.HttpResponse,\n path_parts: list[str],\n server: edbserver.BaseServer,\n tenant: edbtenant.Tenant,\n is_tenant_host: bool,\n) -> None:\n try:\n if tenant is None:\n try:\n tenant = server.get_default_tenant()\n except Exception:\n # Multi-tenant server doesn't have default tenant\n pass\n if tenant is None and not is_tenant_host:\n _response(\n response,\n http.HTTPStatus.NOT_FOUND,\n b'\"No such tenant configured\"',\n True,\n )\n elif path_parts == ['status', 'ready'] and request.method == b'GET':\n if tenant is None:\n await handle_compiler_query(server, response)\n else:\n await tenant.create_task(\n handle_readiness_query(request, response, tenant),\n interruptable=False,\n )\n elif path_parts == ['status', 'alive'] and request.method == b'GET':\n if tenant is None:\n await handle_compiler_query(server, response)\n else:\n await tenant.create_task(\n handle_liveness_query(request, response, tenant),\n interruptable=False,\n )\n else:\n _response(\n response,\n http.HTTPStatus.NOT_FOUND,\n b'\"Unknown path\"',\n True,\n )\n except errors.BackendUnavailableError as ex:\n _response_error(\n response, http.HTTPStatus.SERVICE_UNAVAILABLE, str(ex), type(ex)\n )\n except errors.EdgeDBError as ex:\n if debug.flags.server:\n markup.dump(ex)\n _response_error(\n response, http.HTTPStatus.INTERNAL_SERVER_ERROR, str(ex), type(ex)\n )\n except Exception as ex:\n if debug.flags.server:\n markup.dump(ex)\n\n # XXX Fix this when LSP \"location\" objects are implemented\n ex_type = errors.InternalServerError\n\n _response_error(\n response, http.HTTPStatus.INTERNAL_SERVER_ERROR, str(ex), ex_type\n )\n\n\ndef _response_error(\n response: protocol.HttpResponse,\n status: http.HTTPStatus,\n message: str,\n ex_type: type[errors.EdgeDBError],\n) -> None:\n err_dct = {\n 'message': message,\n 'type': str(ex_type.__name__),\n 'code': ex_type.get_code(),\n }\n _response(response, status, json.dumps({'error': err_dct}).encode(), True)\n\n\ndef _response(\n response: protocol.HttpResponse,\n status: http.HTTPStatus,\n message: bytes,\n close_connection: bool,\n) -> None:\n response.body = message\n response.status = status\n response.content_type = b'application/json'\n response.close_connection = close_connection\n\n\ndef _response_ok(response: protocol.HttpResponse, message: bytes) -> None:\n _response(response, http.HTTPStatus.OK, message, False)\n\n\nasync def _ping(\n response: protocol.HttpResponse, tenant: edbtenant.Tenant\n) -> None:\n try:\n async with asyncio.TaskGroup() as tg:\n ping_backend = tg.create_task(tenant.ping_backend())\n ping_compiler = tg.create_task(\n tenant.server.get_compiler_pool().health_check()\n )\n except *TimeoutError:\n if isinstance(ping_backend.exception(), TimeoutError):\n who = \"the backend\"\n else:\n who = \"the compiler pool\"\n _response_error(\n response,\n http.HTTPStatus.SERVICE_UNAVAILABLE,\n f\"{who} health check timed out\",\n errors.AvailabilityError,\n )\n else:\n if not ping_backend.result():\n _response_error(\n response,\n http.HTTPStatus.SERVICE_UNAVAILABLE,\n \"this server is not ready to accept connections\",\n errors.BackendUnavailableError,\n )\n elif not ping_compiler.result():\n _response_error(\n response,\n http.HTTPStatus.SERVICE_UNAVAILABLE,\n \"The compiler pool is not ready\",\n errors.AvailabilityError,\n )\n else:\n _response_ok(response, b'\"OK\"')\n\n\nasync def handle_compiler_query(\n server: edbserver.BaseServer,\n response: protocol.HttpResponse,\n) -> None:\n if await server.get_compiler_pool().health_check():\n _response_ok(response, b'\"OK\"')\n else:\n _response_error(\n response,\n http.HTTPStatus.SERVICE_UNAVAILABLE,\n \"The compiler pool is not ready\",\n errors.AvailabilityError,\n )\n\n\nasync def handle_liveness_query(\n request: protocol.HttpRequest,\n response: protocol.HttpResponse,\n tenant: edbtenant.Tenant,\n) -> None:\n await _ping(response, tenant)\n\n\nasync def handle_readiness_query(\n request: protocol.HttpRequest,\n response: protocol.HttpResponse,\n tenant: edbtenant.Tenant,\n) -> None:\n if not tenant.is_ready():\n _response_error(\n response,\n http.HTTPStatus.SERVICE_UNAVAILABLE,\n \"this server is not ready to accept connections\",\n errors.AccessError,\n )\n else:\n await _ping(response, tenant)\n" }, { "path": "edb/server/protocol/ui_ext.pyx", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2019-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\nimport base64\nimport http\nimport json\nimport urllib.parse\nimport os\nimport mimetypes\n\nimport immutables\n\nfrom edb import buildmeta\nfrom edb import errors\n\nfrom edb.common import debug\nfrom edb.common import markup\n\n\nSTATIC_FILES_DIR = str(buildmeta.get_shared_data_dir_path() / 'ui')\n\nstatic_files = dict()\n\ndef cache_assets():\n for dirpath, _, filenames in os.walk(STATIC_FILES_DIR):\n for filename in filenames:\n fullpath = os.path.join(dirpath, filename)\n\n mimetype = mimetypes.guess_type(filename)[0]\n if mimetype is None:\n mimetype = 'application/octet-stream'\n\n with open(fullpath, 'rb') as f:\n static_files[os.path.relpath(fullpath, STATIC_FILES_DIR)] = (\n f.read(),\n mimetype.encode()\n )\n\nasync def handle_request(\n request,\n response,\n path_parts,\n server,\n):\n try:\n if path_parts == []:\n path_parts = ['index.html']\n\n data, content_type = static_files.get(\n os.path.join(*path_parts),\n static_files['index.html']\n )\n response.status = http.HTTPStatus.OK\n response.content_type = content_type\n response.body = data\n return\n except Exception as ex:\n return handle_error(request, response, ex)\n\n\ndef handle_error(\n request,\n response,\n error\n):\n if debug.flags.server:\n markup.dump(error)\n\n response.body = b'Internal Server Error'\n response.status = http.HTTPStatus.INTERNAL_SERVER_ERROR\n response.close_connection = True\n" }, { "path": "edb/server/rust_async_channel.py", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2024-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\nimport asyncio\nimport io\nimport logging\n\n\nfrom typing import Protocol, Optional, Any, Callable\n\nlogger = logging.getLogger(\"edb.server\")\n\nMAX_BATCH_SIZE = 16\n\n\nclass RustPipeProtocol(Protocol):\n def _read(self) -> tuple[Any, ...]: ...\n\n def _try_read(self) -> Optional[tuple[Any, ...]]: ...\n\n def _close_pipe(self) -> None: ...\n\n _fd: int\n\n\nclass RustAsyncChannel:\n _buffered_reader: io.BufferedReader\n _skip_reads: int\n _closed: asyncio.Event\n\n def __init__(\n self,\n pipe: RustPipeProtocol,\n callback: Callable[[tuple[Any, ...]], None],\n ) -> None:\n self._closed = asyncio.Event()\n fd = pipe._fd\n self._buffered_reader = io.BufferedReader(\n io.FileIO(fd), buffer_size=MAX_BATCH_SIZE\n )\n self._fd = fd\n self._pipe = pipe\n self._callback = callback\n self._skip_reads = 0\n\n def __del__(self):\n if not self._closed.is_set():\n logger.error(f\"RustAsyncChannel {id(self)} was not closed\")\n\n async def run(self):\n loop = asyncio.get_running_loop()\n loop.add_reader(self._fd, self._channel_read)\n try:\n await self._closed.wait()\n finally:\n loop.remove_reader(self._fd)\n\n def close(self):\n if not self._closed.is_set():\n self._pipe._close_pipe()\n self._buffered_reader.close()\n self._closed.set()\n\n def read_hint(self):\n while msg := self._pipe._try_read():\n self._skip_reads += 1\n self._callback(msg)\n\n def _channel_read(self) -> None:\n try:\n n = len(self._buffered_reader.read1(MAX_BATCH_SIZE))\n if n == 0:\n return\n if self._skip_reads > n:\n self._skip_reads -= n\n return\n n -= self._skip_reads\n self._skip_reads = 0\n for _ in range(n):\n msg = self._pipe._read()\n if msg is None:\n self.close()\n return\n self._callback(msg)\n except Exception:\n logger.error(\n f\"Error reading from Rust async channel\", exc_info=True\n )\n self.close()\n" }, { "path": "edb/server/server.py", "content": "# mypy: check-untyped-defs\n\n#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2016-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nfrom __future__ import annotations\nimport re\nfrom typing import (\n Any,\n Callable,\n Optional,\n Hashable,\n Iterator,\n Mapping,\n Sequence,\n TYPE_CHECKING,\n)\n\nimport asyncio\nimport collections\nimport ipaddress\nimport itertools\nimport json\nimport logging\nimport os\nimport pathlib\nimport pickle\nimport socket\nimport ssl\nimport stat\nimport time\nimport uuid\n\nimport immutables\n\nfrom edb import buildmeta\nfrom edb import errors\n\nfrom edb.common import devmode\nfrom edb.common import lru\nfrom edb.common import secretkey\nfrom edb.common import windowedsum\nfrom edb.common.log import current_tenant\n\nfrom edb.schema import reflection as s_refl\nfrom edb.schema import schema as s_schema\n\nfrom edb.server import auth\nfrom edb.server import args as srvargs\nfrom edb.server import cache\nfrom edb.server import config\nfrom edb.server import compiler_pool\nfrom edb.server import daemon\nfrom edb.server import defines\nfrom edb.server import instdata\nfrom edb.server import protocol\nfrom edb.server import net_worker\nfrom edb.server import tenant as edbtenant\nfrom edb.server.protocol import binary # type: ignore\nfrom edb.server.protocol import pg_ext # type: ignore\nfrom edb.server.protocol import ui_ext # type: ignore\nfrom edb.server.protocol.auth_ext import pkce\nfrom edb.server import metrics\nfrom edb.server import pgcon\n\nfrom edb.pgsql import patches as pg_patches\n\nfrom . import compiler as edbcompiler\nfrom .compiler import sertypes\n\nif TYPE_CHECKING:\n import asyncio.base_events\n\n from edb.pgsql import params as pgparams\n\n from . import bootstrap\n\n\nADMIN_PLACEHOLDER = \"\"\nlogger = logging.getLogger('edb.server')\nlog_metrics = logging.getLogger('edb.server.metrics')\n\n\nclass StartupError(Exception):\n pass\n\n\nclass BaseServer:\n _sys_queries: Mapping[str, bytes]\n _local_intro_query: bytes\n _global_intro_query: bytes\n _report_config_typedesc: dict[defines.ProtocolVersion, bytes]\n _use_monitor_fs: bool\n _file_watch_handles: list[asyncio.Handle]\n\n _std_schema: s_schema.Schema\n _refl_schema: s_schema.Schema\n _schema_class_layout: s_refl.SchemaClassLayout\n\n _servers: Mapping[str, asyncio.AbstractServer]\n\n _testmode: bool\n\n # We maintain an OrderedDict of all active client connections.\n # We use an OrderedDict because it allows to move keys to either\n # end of the dict. That's used to keep all active client connections\n # grouped at the right end of the dict. The idea is that we can then\n # have a periodically run coroutine to GC all inactive connections.\n # This should be more economical than maintaining a TimerHandle for\n # every open connection. Also, this way, we can react to the\n # `session_idle_timeout` config setting changed mid-flight.\n _binary_conns: collections.OrderedDict[binary.EdgeConnection, bool]\n _pgext_conns: dict[str, pg_ext.PgConnection]\n _idle_gc_handler: asyncio.TimerHandle | None = None\n _stmt_cache_size: int | None = None\n\n _compiler_pool: compiler_pool.AbstractPool | None\n compilation_config_serializer: sertypes.CompilationConfigSerializer\n _http_request_logger: asyncio.Task | None\n _auth_gc: asyncio.Task | None\n _net_worker_http: asyncio.Task | None\n _net_worker_http_gc: asyncio.Task | None\n\n def __init__(\n self,\n *,\n runstate_dir: pathlib.Path,\n internal_runstate_dir: pathlib.Path,\n compiler_pool_size: int,\n compiler_worker_branch_limit,\n compiler_pool_mode: srvargs.CompilerPoolMode,\n compiler_pool_addr: tuple[str, int],\n nethosts: Sequence[str],\n netport: int,\n compiler_worker_max_rss: Optional[int] = None,\n listen_sockets: tuple[socket.socket, ...] = (),\n testmode: bool = False,\n daemonized: bool = False,\n pidfile_dir: Optional[pathlib.Path] = None,\n binary_endpoint_security: srvargs.ServerEndpointSecurityMode = (\n srvargs.ServerEndpointSecurityMode.Tls),\n http_endpoint_security: srvargs.ServerEndpointSecurityMode = (\n srvargs.ServerEndpointSecurityMode.Tls),\n auto_shutdown_after: float = -1,\n echo_runtime_info: bool = False,\n status_sinks: Sequence[Callable[[str], None]] = (),\n default_auth_method: srvargs.ServerAuthMethods = (\n srvargs.DEFAULT_AUTH_METHODS),\n admin_ui: bool = False,\n cors_always_allowed_origins: Optional[str] = None,\n disable_dynamic_system_config: bool = False,\n compiler_state: edbcompiler.CompilerState,\n use_monitor_fs: bool = False,\n net_worker_mode: srvargs.NetWorkerMode = srvargs.NetWorkerMode.Default,\n ):\n self.__loop = asyncio.get_running_loop()\n self._use_monitor_fs = use_monitor_fs\n\n self._schema_class_layout = compiler_state.schema_class_layout\n self._config_settings = compiler_state.config_spec\n self._refl_schema = compiler_state.refl_schema\n self._std_schema = compiler_state.std_schema\n assert compiler_state.global_intro_query is not None\n self._global_intro_query = (\n compiler_state.global_intro_query.encode(\"utf-8\"))\n assert compiler_state.local_intro_query is not None\n self._local_intro_query = (\n compiler_state.local_intro_query.encode(\"utf-8\"))\n\n # Used to tag PG notifications to later disambiguate them.\n self._server_id = str(uuid.uuid4())\n\n self._daemonized = daemonized\n self._pidfile_dir = pidfile_dir\n self._runstate_dir = runstate_dir\n self._internal_runstate_dir = internal_runstate_dir\n self._compiler_pool = None\n self._compiler_pool_size = compiler_pool_size\n self._compiler_worker_branch_limit = compiler_worker_branch_limit\n self._compiler_pool_mode = compiler_pool_mode\n self._compiler_pool_addr = compiler_pool_addr\n self._compiler_worker_max_rss = compiler_worker_max_rss\n self._system_compile_cache = lru.LRUMapping(\n maxsize=defines._MAX_QUERIES_CACHE_SYSTEM\n )\n self._system_compile_cache_locks: dict[Any, Any] = {}\n\n self._listen_sockets = listen_sockets\n if listen_sockets:\n nethosts = tuple(s.getsockname()[0] for s in listen_sockets)\n netport = listen_sockets[0].getsockname()[1]\n\n self._listen_hosts = nethosts\n self._listen_port = netport\n\n # Shutdown the server after the last management\n # connection has disconnected\n # and there have been no new connections for n seconds\n self._auto_shutdown_after = auto_shutdown_after\n self._auto_shutdown_handler: Any = None\n self._keepalive_tokens: set = set()\n\n self._echo_runtime_info = echo_runtime_info\n self._status_sinks = status_sinks\n\n self._sys_queries = immutables.Map()\n\n self._devmode = devmode.is_in_dev_mode()\n self._testmode = testmode\n\n self._binary_proto_id_counter = 0\n self._binary_conns = collections.OrderedDict()\n self._pgext_conns = {}\n\n self._servers = {}\n\n self._http_query_cache = cache.StatementsCache(\n maxsize=defines.HTTP_PORT_QUERY_CACHE_SIZE)\n\n self._http_last_minute_requests = windowedsum.WindowedSum()\n self._http_request_logger = None\n self._auth_gc = None\n self._net_worker_http = None\n self._net_worker_http_gc = None\n self._net_worker_mode = net_worker_mode\n\n self._stop_evt = asyncio.Event()\n self._tls_cert_file: str | Any = None\n self._tls_cert_newly_generated = False\n self._sslctx: ssl.SSLContext | Any = None\n self._sslctx_pgext: ssl.SSLContext | Any = None\n\n self._jws_key: auth.JWKSet | None = None\n self._jws_keys_newly_generated = False\n\n self._default_auth_method_spec = default_auth_method\n self._default_auth_methods = self._get_auth_method_types(\n default_auth_method)\n self._binary_endpoint_security = binary_endpoint_security\n self._http_endpoint_security = http_endpoint_security\n\n self._idle_gc_handler = None\n\n self._admin_ui = admin_ui\n\n self._cors_always_allowed_origins = [\n re.compile(\n '^' + origin\n .replace('.', '\\\\.')\n .replace('*', '.*') + '$'\n ) if '*' in origin else origin\n for origin in cors_always_allowed_origins.split(',')\n ] if cors_always_allowed_origins else []\n\n self._file_watch_handles = []\n self._tls_certs_reload_retry_handle: Any | asyncio.TimerHandle = None\n\n self._disable_dynamic_system_config = disable_dynamic_system_config\n self._report_config_typedesc = {}\n\n def _get_auth_method_types(\n self,\n auth_methods_spec: srvargs.ServerAuthMethods,\n ) -> dict[srvargs.ServerConnTransport, list[config.CompositeConfigType]]:\n mapping = {}\n for transport, methods in auth_methods_spec.items():\n result = []\n for method in methods:\n auth_type = self.config_settings.get_type_by_name(\n f'cfg::{method.value}'\n )\n result.append(auth_type())\n mapping[transport] = result\n\n return mapping\n\n async def _request_stats_logger(self):\n last_seen = -1\n while True:\n current = int(self._http_last_minute_requests)\n if current != last_seen:\n log_metrics.info(\n \"HTTP requests in last minute: %d\",\n current,\n )\n last_seen = current\n\n await asyncio.sleep(30)\n\n def get_server_id(self):\n return self._server_id\n\n def get_listen_hosts(self):\n return self._listen_hosts\n\n def get_listen_port(self):\n return self._listen_port\n\n def get_loop(self):\n return self.__loop\n\n def in_dev_mode(self):\n return self._devmode\n\n def in_test_mode(self):\n return self._testmode\n\n def is_admin_ui_enabled(self):\n return self._admin_ui\n\n def get_cors_always_allowed_origins(self):\n return self._cors_always_allowed_origins\n\n def on_binary_client_created(self) -> str:\n self._binary_proto_id_counter += 1\n return str(self._binary_proto_id_counter)\n\n def on_binary_client_connected(self, conn):\n self._binary_conns[conn] = True\n metrics.current_client_connections.inc(\n 1.0, conn.get_tenant_label()\n )\n\n def on_binary_client_authed(self, conn):\n self._report_connections(event='opened')\n metrics.total_client_connections.inc(\n 1.0, conn.get_tenant_label()\n )\n\n def on_binary_client_after_idling(self, conn):\n try:\n self._binary_conns.move_to_end(conn, last=True)\n except KeyError:\n # Shouldn't happen, but just in case some weird async twist\n # gets us here we don't want to crash the connection with\n # this error.\n metrics.background_errors.inc(\n 1.0, conn.get_tenant_label(), 'client_after_idling'\n )\n\n def on_binary_client_disconnected(self, conn):\n self._binary_conns.pop(conn, None)\n self._report_connections(event=\"closed\")\n metrics.current_client_connections.dec(\n 1.0, conn.get_tenant_label()\n )\n self.maybe_auto_shutdown()\n\n def maybe_delay_auto_shutdown(self):\n if self._auto_shutdown_handler:\n self._auto_shutdown_handler.cancel()\n self._auto_shutdown_handler = None\n\n def maybe_auto_shutdown(self):\n if (\n not self._binary_conns\n and not self._keepalive_tokens\n and self._auto_shutdown_after >= 0\n and self._auto_shutdown_handler is None\n ):\n self._auto_shutdown_handler = self.__loop.call_later(\n self._auto_shutdown_after, self.request_auto_shutdown)\n\n def _report_connections(self, *, event: str) -> None:\n log_metrics.info(\n \"%s a connection; open_count=%d\",\n event,\n len(self._binary_conns),\n )\n\n def add_keepalive_token(self, token: Hashable) -> None:\n self.maybe_delay_auto_shutdown()\n self._keepalive_tokens.add(token)\n\n def remove_keepalive_token(self, token: Hashable) -> None:\n self._keepalive_tokens.discard(token)\n self.maybe_auto_shutdown()\n\n def on_pgext_client_connected(self, conn):\n self._pgext_conns[conn.get_id()] = conn\n\n def on_pgext_client_disconnected(self, conn):\n self._pgext_conns.pop(conn.get_id(), None)\n self.maybe_auto_shutdown()\n\n def cancel_pgext_connection(self, pid, secret):\n conn = self._pgext_conns.get(pid)\n if conn is not None:\n conn.cancel(secret)\n\n def monitor_fs(\n self,\n file_path: str | pathlib.Path,\n cb: Callable[[], None],\n ) -> Callable[[], None]:\n if not self._use_monitor_fs:\n return lambda: None\n\n if isinstance(file_path, str):\n path = pathlib.Path(file_path)\n path_str = file_path\n else:\n path = file_path\n path_str = str(file_path)\n handle = None\n parent_dir = path.parent\n\n def watch_dir(file_modified, _event):\n nonlocal handle\n if parent_dir / os.fsdecode(file_modified) == path:\n try:\n new_handle = self.__loop._monitor_fs( # type: ignore\n path_str, callback)\n except FileNotFoundError:\n pass\n else:\n finalizer()\n handle = new_handle\n self._file_watch_handles.append(handle)\n cb()\n\n def callback(_file_modified, _event):\n nonlocal handle\n # First, cancel the existing watcher and call cb() regardless of\n # what event it is. This is because macOS issues RENAME while Linux\n # issues CHANGE, and we don't have enough knowledge about renaming.\n # The idea here is to re-watch the file path after every event, so\n # that even if the file is recreated, we still watch the right one.\n finalizer()\n try:\n cb()\n finally:\n try:\n # Then, see if we can directly re-watch the target path\n handle = self.__loop._monitor_fs( # type: ignore\n path_str, callback)\n except FileNotFoundError:\n # If not, watch the parent directory to wait for recreation\n handle = self.__loop._monitor_fs( # type: ignore\n str(parent_dir), watch_dir)\n self._file_watch_handles.append(handle)\n\n # ... we depend on an event loop internal _monitor_fs\n handle = self.__loop._monitor_fs(path_str, callback) # type: ignore\n\n def finalizer():\n try:\n self._file_watch_handles.remove(handle)\n except ValueError:\n # The server may have cleared _file_watch_handles before the\n # tenants do, so we can skip the double cancel here.\n pass\n else:\n handle.cancel()\n\n self._file_watch_handles.append(handle)\n\n return finalizer\n\n def _get_sys_config(self) -> Mapping[str, config.SettingValue]:\n raise NotImplementedError\n\n def config_lookup(\n self,\n name: str,\n *configs: Mapping[str, config.SettingValue],\n ) -> Any:\n return config.lookup(name, *configs, spec=self._config_settings)\n\n @property\n def config_settings(self) -> config.Spec:\n return self._config_settings\n\n async def init(self):\n if self.is_admin_ui_enabled():\n ui_ext.cache_assets()\n\n sys_config = self._get_sys_config()\n if not self._listen_hosts:\n self._listen_hosts = (\n self.config_lookup('listen_addresses', sys_config)\n or ('localhost',)\n )\n\n if self._listen_port is None:\n self._listen_port = (\n self.config_lookup('listen_port', sys_config)\n or defines.EDGEDB_PORT\n )\n\n self._stmt_cache_size = self.config_lookup(\n '_pg_prepared_statement_cache_size', sys_config\n )\n\n self.reinit_idle_gc_collector()\n\n def reinit_idle_gc_collector(self) -> float:\n if self._auto_shutdown_after >= 0:\n return -1\n\n if self._idle_gc_handler is not None:\n self._idle_gc_handler.cancel()\n self._idle_gc_handler = None\n\n session_idle_timeout = self.config_lookup(\n 'session_idle_timeout', self._get_sys_config())\n\n timeout = session_idle_timeout.to_microseconds()\n timeout /= 1_000_000.0 # convert to seconds\n\n if timeout > 0:\n self._idle_gc_handler = self.__loop.call_later(\n timeout, self._idle_gc_collector)\n\n return timeout\n\n @property\n def stmt_cache_size(self) -> int | None:\n return self._stmt_cache_size\n\n @property\n def system_compile_cache(self):\n return self._system_compile_cache\n\n def request_stop_fe_conns(self, dbname: str) -> None:\n for conn in itertools.chain(\n self._binary_conns.keys(), self._pgext_conns.values()\n ):\n if conn.dbname == dbname:\n conn.request_stop()\n\n @property\n def system_compile_cache_locks(self):\n return self._system_compile_cache_locks\n\n def _idle_gc_collector(self):\n try:\n self._idle_gc_handler = None\n idle_timeout = self.reinit_idle_gc_collector()\n\n if idle_timeout <= 0:\n return\n\n now = time.monotonic()\n expiry_time = now - idle_timeout\n for conn in self._binary_conns:\n try:\n if conn.is_idle(expiry_time):\n label = conn.get_tenant_label()\n metrics.idle_client_connections.inc(1.0, label)\n current_tenant.set(label)\n conn.close_for_idling()\n elif conn.is_alive():\n # We are sorting connections in\n # 'on_binary_client_after_idling' to specifically\n # enable this optimization. As soon as we find first\n # non-idle active connection we're guaranteed\n # to have traversed all of the potentially idling\n # connections.\n break\n except Exception:\n metrics.background_errors.inc(\n 1.0, conn.get_tenant_label(), 'close_for_idling'\n )\n conn.abort()\n except Exception:\n metrics.background_errors.inc(\n 1.0, 'system', 'idle_clients_collector'\n )\n raise\n\n def _get_backend_runtime_params(self) -> pgparams.BackendRuntimeParams:\n raise NotImplementedError\n\n def _get_compiler_args(self) -> dict[str, Any]:\n # Force Postgres version in BackendRuntimeParams to be the\n # minimal supported, because the compiler does not rely on\n # the version, and not pinning it would make the remote compiler\n # pool refuse connections from clients that have differing versions\n # of Postgres backing them.\n runtime_params = self._get_backend_runtime_params()\n min_ver = '.'.join(str(v) for v in defines.MIN_POSTGRES_VERSION)\n runtime_params = runtime_params._replace(\n instance_params=runtime_params.instance_params._replace(\n version=buildmeta.parse_pg_version(min_ver),\n ),\n )\n\n args = dict(\n pool_size=self._compiler_pool_size,\n worker_branch_limit=self._compiler_worker_branch_limit,\n pool_class=self._compiler_pool_mode.pool_class,\n runstate_dir=self._internal_runstate_dir,\n backend_runtime_params=runtime_params,\n std_schema=self._std_schema,\n refl_schema=self._refl_schema,\n schema_class_layout=self._schema_class_layout,\n )\n if self._compiler_pool_mode == srvargs.CompilerPoolMode.Remote:\n args['address'] = self._compiler_pool_addr\n else:\n if self._compiler_worker_max_rss is not None:\n args['worker_max_rss'] = self._compiler_worker_max_rss\n return args\n\n async def _destroy_compiler_pool(self):\n if self._compiler_pool is not None:\n await self._compiler_pool.stop()\n self._compiler_pool = None\n\n def get_compiler_pool(self):\n return self._compiler_pool\n\n async def introspect_global_schema_json(\n self, conn: pgcon.PGConnection\n ) -> bytes:\n return await conn.sql_fetch_val(self._global_intro_query)\n\n def _parse_global_schema(\n self, json_data: Any\n ) -> s_schema.Schema:\n return s_refl.parse_schema(\n base_schema=self._std_schema,\n data=json_data,\n schema_class_layout=self._schema_class_layout,\n )\n\n async def introspect_global_schema(\n self, conn: pgcon.PGConnection\n ) -> s_schema.Schema:\n json_data = await self.introspect_global_schema_json(conn)\n return self._parse_global_schema(json_data)\n\n async def introspect_user_schema_json(\n self,\n conn: pgcon.PGConnection,\n ) -> bytes:\n return await conn.sql_fetch_val(self._local_intro_query)\n\n def _parse_user_schema(\n self,\n json_data: Any,\n global_schema: s_schema.Schema,\n ) -> s_schema.Schema:\n base_schema = s_schema.ChainedSchema(\n self._std_schema,\n s_schema.EMPTY_SCHEMA,\n global_schema,\n )\n\n return s_refl.parse_schema(\n base_schema=base_schema,\n data=json_data,\n schema_class_layout=self._schema_class_layout,\n )\n\n async def _introspect_user_schema(\n self,\n conn: pgcon.PGConnection,\n global_schema: s_schema.Schema,\n ) -> s_schema.Schema:\n json_data = await self.introspect_user_schema_json(conn)\n return self._parse_user_schema(json_data, global_schema)\n\n async def introspect_db_config(self, conn: pgcon.PGConnection) -> bytes:\n return await conn.sql_fetch_val(self.get_sys_query(\"dbconfig\"))\n\n def _parse_db_config(\n self, db_config_json: bytes, user_schema: s_schema.Schema\n ) -> Mapping[str, config.SettingValue]:\n spec = config.ChainedSpec(\n self._config_settings,\n config.load_ext_spec_from_schema(\n user_schema,\n self.get_std_schema(),\n ),\n )\n\n return config.from_json(spec, db_config_json)\n\n async def get_dbnames(self, syscon):\n dbs_query = self.get_sys_query('listdbs')\n json_data = await syscon.sql_fetch_val(dbs_query)\n return json.loads(json_data)\n\n async def _on_system_config_add(self, setting_name, value):\n # CONFIGURE INSTANCE INSERT ConfigObject;\n pass\n\n async def _on_system_config_rem(self, setting_name, value):\n # CONFIGURE INSTANCE RESET ConfigObject;\n pass\n\n async def _on_system_config_set(self, setting_name, value):\n # CONFIGURE INSTANCE SET setting_name := value;\n pass\n\n async def _on_system_config_reset(self, setting_name):\n # CONFIGURE INSTANCE RESET setting_name;\n pass\n\n def before_alter_system_config(self):\n if self._disable_dynamic_system_config:\n raise errors.ConfigurationError(\n \"cannot change this configuration value in this instance\"\n )\n\n async def _after_system_config_add(self, setting_name, value):\n # CONFIGURE INSTANCE INSERT ConfigObject;\n pass\n\n async def _after_system_config_rem(self, setting_name, value):\n # CONFIGURE INSTANCE RESET ConfigObject;\n pass\n\n async def _after_system_config_set(self, setting_name, value):\n # CONFIGURE INSTANCE SET setting_name := value;\n pass\n\n async def _after_system_config_reset(self, setting_name):\n # CONFIGURE INSTANCE RESET setting_name;\n pass\n\n def _make_protocol(self):\n self.maybe_delay_auto_shutdown()\n return protocol.HttpProtocol(\n self,\n self._sslctx,\n self._sslctx_pgext,\n binary_endpoint_security=self._binary_endpoint_security,\n http_endpoint_security=self._http_endpoint_security,\n )\n\n async def _start_server(\n self,\n host: str,\n port: int,\n sock: Optional[socket.socket] = None,\n ) -> Optional[asyncio.base_events.Server]:\n try:\n kwargs: dict[str, Any]\n if sock is not None:\n kwargs = {\"sock\": sock}\n else:\n kwargs = {\"host\": host, \"port\": port}\n return await self.__loop.create_server(\n self._make_protocol, **kwargs\n )\n except Exception as e:\n logger.warning(\n f\"could not create listen socket for '{host}:{port}': {e}\"\n )\n return None\n\n async def _start_admin_server(\n self,\n port: int,\n ) -> asyncio.base_events.Server:\n admin_unix_sock_path = os.path.join(\n self._runstate_dir, f'.s.GEL.admin.{port}')\n symlink = os.path.join(\n self._runstate_dir, f'.s.EDGEDB.admin.{port}')\n\n exists = False\n try:\n mode = os.lstat(symlink).st_mode\n if stat.S_ISSOCK(mode):\n os.unlink(symlink)\n else:\n exists = True\n except FileNotFoundError:\n pass\n if not exists:\n os.symlink(admin_unix_sock_path, symlink)\n\n assert len(admin_unix_sock_path) <= (\n defines.MAX_RUNSTATE_DIR_PATH\n + defines.MAX_UNIX_SOCKET_PATH_LENGTH\n + 1\n ), \"admin Unix socket length exceeds maximum allowed\"\n admin_unix_srv = await self.__loop.create_unix_server(\n lambda: binary.new_edge_connection(\n self, self._get_admin_tenant(), external_auth=True\n ),\n admin_unix_sock_path\n )\n os.chmod(admin_unix_sock_path, stat.S_IRUSR | stat.S_IWUSR)\n logger.info('Serving admin on %s', admin_unix_sock_path)\n return admin_unix_srv\n\n def _get_admin_tenant(self) -> edbtenant.Tenant:\n return self.get_default_tenant()\n\n async def _start_servers(\n self,\n hosts: tuple[str, ...],\n port: int,\n *,\n admin: bool = True,\n sockets: tuple[socket.socket, ...] = (),\n ):\n servers = {}\n if port == 0:\n # Automatic port selection requires us to start servers\n # sequentially until we get a working bound socket to ensure\n # consistent port value across all requested listen addresses.\n try:\n for host in hosts:\n server = await self._start_server(host, port)\n if server is not None:\n if port == 0:\n port = server.sockets[0].getsockname()[1]\n servers[host] = server\n except Exception:\n await self._stop_servers(servers.values())\n raise\n else:\n start_tasks = {}\n try:\n async with asyncio.TaskGroup() as g:\n if sockets:\n for host, sock in zip(hosts, sockets):\n start_tasks[host] = g.create_task(\n self._start_server(host, port, sock=sock)\n )\n else:\n for host in hosts:\n start_tasks[host] = g.create_task(\n self._start_server(host, port)\n )\n except Exception:\n await self._stop_servers([\n fut.result() for fut in start_tasks.values()\n if (\n fut.done()\n and fut.exception() is None\n and fut.result() is not None\n )\n ])\n raise\n\n servers.update({\n host: srv\n for host, fut in start_tasks.items()\n if (srv := fut.result()) is not None\n })\n\n # Fail if none of the servers can be started, except when the admin\n # server on a UNIX domain socket will be started.\n if not servers and (not admin or port == 0):\n raise StartupError(\"could not create any listen sockets\")\n\n addrs = []\n for tcp_srv in servers.values():\n for s in tcp_srv.sockets:\n addrs.append(s.getsockname())\n\n if len(addrs) > 1:\n if port:\n addr_str = f\"{{{', '.join(addr[0] for addr in addrs)}}}:{port}\"\n else:\n addr_str = f\"\"\"{{{', '.join(\n f'{addr[0]}:{addr[1]}' for addr in addrs)}}}\"\"\"\n elif addrs:\n addr_str = f'{addrs[0][0]}:{addrs[0][1]}'\n port = addrs[0][1]\n else:\n addr_str = None\n\n if addr_str:\n logger.info('Serving on %s', addr_str)\n\n if admin and port:\n try:\n admin_unix_srv = await self._start_admin_server(port)\n except Exception:\n await self._stop_servers(servers.values())\n raise\n servers[ADMIN_PLACEHOLDER] = admin_unix_srv\n\n return servers, port, addrs\n\n def _sni_callback(self, sslobj, server_name, sslctx):\n # Match the given SNI for a pre-registered Tenant instance,\n # and temporarily store in memory indexed by sslobj for future\n # retrieval, see also retrieve_tenant() below.\n #\n # Used in multi-tenant server only. This method must not fail.\n pass\n\n def reload_tls(self, tls_cert_file, tls_key_file, client_ca_file):\n logger.info(\"loading TLS certificates\")\n tls_password_needed = False\n if self._tls_certs_reload_retry_handle is not None:\n self._tls_certs_reload_retry_handle.cancel()\n self._tls_certs_reload_retry_handle = None\n\n def _tls_private_key_password():\n nonlocal tls_password_needed\n tls_password_needed = True\n return (\n os.environ.get('GEL_SERVER_TLS_PRIVATE_KEY_PASSWORD', '')\n or os.environ.get('EDGEDB_SERVER_TLS_PRIVATE_KEY_PASSWORD', '')\n )\n\n sslctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)\n sslctx_pgext = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)\n try:\n sslctx.load_cert_chain(\n tls_cert_file,\n tls_key_file,\n password=_tls_private_key_password,\n )\n sslctx_pgext.load_cert_chain(\n tls_cert_file,\n tls_key_file,\n password=_tls_private_key_password,\n )\n except ssl.SSLError as e:\n if e.library == \"SSL\" and e.errno == 9: # ERR_LIB_PEM\n if tls_password_needed:\n if _tls_private_key_password():\n raise StartupError(\n \"Cannot load TLS certificates - it's likely that \"\n \"the private key password is wrong.\"\n ) from e\n else:\n raise StartupError(\n \"Cannot load TLS certificates - the private key \"\n \"file is likely protected by a password. Specify \"\n \"the password using environment variable: \"\n \"GEL_SERVER_TLS_PRIVATE_KEY_PASSWORD\"\n ) from e\n elif tls_key_file is None:\n raise StartupError(\n \"Cannot load TLS certificates - have you specified \"\n \"the private key file using the `--tls-key-file` \"\n \"command-line argument?\"\n ) from e\n else:\n raise StartupError(\n \"Cannot load TLS certificates - please double check \"\n \"if the specified certificate files are valid.\"\n )\n elif e.library == \"X509\" and e.errno == 116:\n # X509 Error 116: X509_R_KEY_VALUES_MISMATCH\n raise StartupError(\n \"Cannot load TLS certificates - the private key doesn't \"\n \"match the certificate.\"\n )\n\n raise StartupError(f\"Cannot load TLS certificates - {e}\") from e\n\n if client_ca_file is not None:\n try:\n sslctx.load_verify_locations(client_ca_file)\n sslctx_pgext.load_verify_locations(client_ca_file)\n except ssl.SSLError as e:\n raise StartupError(\n f\"Cannot load client CA certificates - {e}\") from e\n sslctx.verify_mode = ssl.CERT_OPTIONAL\n sslctx_pgext.verify_mode = ssl.CERT_OPTIONAL\n\n sslctx.set_alpn_protocols(['edgedb-binary', 'http/1.1'])\n sslctx.sni_callback = self._sni_callback\n sslctx_pgext.sni_callback = self._sni_callback\n self._sslctx = sslctx\n self._sslctx_pgext = sslctx_pgext\n\n def init_tls(\n self,\n tls_cert_file,\n tls_key_file,\n tls_cert_newly_generated,\n client_ca_file,\n ):\n assert self._sslctx is self._sslctx_pgext is None\n self.reload_tls(tls_cert_file, tls_key_file, client_ca_file)\n\n self._tls_cert_file = str(tls_cert_file)\n self._tls_cert_newly_generated = tls_cert_newly_generated\n\n def reload_tls(retry=0):\n try:\n self.reload_tls(tls_cert_file, tls_key_file, client_ca_file)\n except (StartupError, FileNotFoundError) as e:\n if retry > defines._TLS_CERT_RELOAD_MAX_RETRIES:\n logger.critical(str(e))\n self.request_shutdown()\n else:\n delay = defines._TLS_CERT_RELOAD_EXP_INTERVAL * 2 ** retry\n logger.warning(\"%s; retrying in %.1f seconds.\", e, delay)\n self._tls_certs_reload_retry_handle = (\n self.__loop.call_later(\n delay,\n reload_tls,\n retry + 1,\n )\n )\n except Exception:\n logger.critical(\n \"error while reloading TLS certificate and/or key, \"\n \"shutting down.\",\n exc_info=True,\n )\n self.request_shutdown()\n\n self.monitor_fs(tls_cert_file, reload_tls)\n if tls_cert_file != tls_key_file:\n self.monitor_fs(tls_key_file, reload_tls)\n if client_ca_file is not None:\n self.monitor_fs(client_ca_file, reload_tls)\n\n def start_watching_files(self):\n # TODO(fantix): include the monitor_fs() lines above\n pass\n\n def load_jwcrypto(self, jws_key_file: pathlib.Path) -> auth.JWKSet:\n try:\n jws_key = auth.load_secret_key(jws_key_file)\n self._jws_key = jws_key\n return jws_key\n except auth.SecretKeyReadError as e:\n raise StartupError(e.args[0]) from e\n\n def init_jwcrypto(\n self,\n jws_key_file: pathlib.Path,\n jws_keys_newly_generated: bool,\n ) -> None:\n self.load_jwcrypto(jws_key_file)\n self._jws_keys_newly_generated = jws_keys_newly_generated\n\n def get_jws_key(self) -> auth.JWKSet | None:\n return self._jws_key\n\n async def _stop_servers(self, servers):\n async with asyncio.TaskGroup() as g:\n for srv in servers:\n srv.close()\n g.create_task(srv.wait_closed())\n\n async def _before_start_servers(self) -> None:\n pass\n\n async def _after_start_servers(self) -> None:\n pass\n\n async def start(self):\n self._stop_evt.clear()\n\n self._http_request_logger = self.__loop.create_task(\n self._request_stats_logger()\n )\n\n pool = await compiler_pool.create_compiler_pool(\n **self._get_compiler_args()\n )\n self.compilation_config_serializer = (\n await pool.make_compilation_config_serializer()\n )\n self._compiler_pool = pool\n\n await self._before_start_servers()\n self._servers, actual_port, listen_addrs = await self._start_servers(\n tuple((await _resolve_interfaces(self._listen_hosts))[0]),\n self._listen_port,\n sockets=self._listen_sockets,\n )\n self._listen_hosts = [addr[0] for addr in listen_addrs]\n self._listen_port = actual_port\n\n if self._daemonized:\n pidfile_dir = self._pidfile_dir\n if pidfile_dir is None:\n pidfile_dir = self._runstate_dir\n pidfile_path = pidfile_dir / f\".s.EDGEDB.{actual_port}.lock\"\n pidfile = daemon.PidFile(pidfile_path)\n pidfile.acquire()\n\n await self._after_start_servers()\n self._auth_gc = self.__loop.create_task(pkce.gc(self))\n if self._net_worker_mode is srvargs.NetWorkerMode.Default:\n self._net_worker_http = self.__loop.create_task(\n net_worker.http(self)\n )\n self._net_worker_http_gc = self.__loop.create_task(\n net_worker.gc(self)\n )\n\n if self._echo_runtime_info:\n ri = {\n \"port\": self._listen_port,\n \"runstate_dir\": str(self._runstate_dir),\n \"tls_cert_file\": self._tls_cert_file,\n }\n print(f'\\nEDGEDB_SERVER_DATA:{json.dumps(ri)}\\n', flush=True)\n\n status = self._get_status()\n status[\"listen_addrs\"] = listen_addrs\n status_str = f'READY={json.dumps(status)}'\n for status_sink in self._status_sinks:\n status_sink(status_str)\n\n if self._auto_shutdown_after > 0:\n self._auto_shutdown_handler = self.__loop.call_later(\n self._auto_shutdown_after, self.request_auto_shutdown)\n\n def _get_status(self) -> dict[str, Any]:\n return {\n \"port\": self._listen_port,\n \"socket_dir\": str(self._runstate_dir),\n \"main_pid\": os.getpid(),\n \"tls_cert_file\": self._tls_cert_file,\n \"tls_cert_newly_generated\": self._tls_cert_newly_generated,\n \"jws_keys_newly_generated\": self._jws_keys_newly_generated,\n }\n\n def request_auto_shutdown(self):\n if self._auto_shutdown_after == 0:\n logger.info(\"shutting down server: all clients disconnected\")\n else:\n logger.info(\n f\"shutting down server: no clients connected in last\"\n f\" {self._auto_shutdown_after} seconds\"\n )\n self.request_shutdown()\n\n def request_shutdown(self):\n self._stop_evt.set()\n\n async def stop(self):\n if self._idle_gc_handler is not None:\n self._idle_gc_handler.cancel()\n self._idle_gc_handler = None\n\n if self._http_request_logger is not None:\n self._http_request_logger.cancel()\n if self._auth_gc is not None:\n self._auth_gc.cancel()\n if self._net_worker_http is not None:\n self._net_worker_http.cancel()\n if self._net_worker_http_gc is not None:\n self._net_worker_http_gc.cancel()\n\n for handle in self._file_watch_handles:\n handle.cancel()\n self._file_watch_handles.clear()\n\n await self._stop_servers(self._servers.values())\n self._servers = {}\n\n # This should be done by tenant.stop(), but let's still do it again\n for conn in self._binary_conns:\n conn.request_stop()\n self._binary_conns.clear()\n\n for conn in self._pgext_conns.values():\n conn.request_stop()\n self._pgext_conns.clear()\n\n def request_frontend_stop(self, tenant: edbtenant.Tenant):\n dropped = []\n for conn in self._binary_conns:\n if conn.tenant is tenant:\n conn.request_stop()\n dropped.append(conn)\n for conn in dropped:\n self._binary_conns.pop(conn, None)\n\n dropped.clear()\n for conn in self._pgext_conns.values():\n if conn.tenant is tenant:\n conn.request_stop()\n dropped.append(conn)\n for conn in dropped:\n self._pgext_conns.pop(conn, None)\n\n async def serve_forever(self):\n await self._stop_evt.wait()\n\n def get_sys_query(self, key):\n return self._sys_queries[key]\n\n def get_debug_info(self):\n \"\"\"Used to render the /server-info endpoint in dev/test modes.\n\n Some tests depend on the exact layout of the returned structure.\n \"\"\"\n\n return dict(\n params=dict(\n dev_mode=self._devmode,\n test_mode=self._testmode,\n default_auth_methods=str(self._default_auth_method_spec),\n listen_hosts=self._listen_hosts,\n listen_port=self._listen_port,\n ),\n instance_config=config.debug_serialize_config(\n self._get_sys_config()),\n compiler_pool=(\n self._compiler_pool.get_debug_info()\n if self._compiler_pool\n else None\n ),\n )\n\n def get_report_config_typedesc(\n self,\n ) -> dict[defines.ProtocolVersion, bytes]:\n return self._report_config_typedesc\n\n def get_default_auth_methods(\n self, transport: srvargs.ServerConnTransport\n ) -> list[config.CompositeConfigType]:\n return self._default_auth_methods.get(transport, [])\n\n def get_std_schema(self) -> s_schema.Schema:\n return self._std_schema\n\n def retrieve_tenant(self, sslobj) -> edbtenant.Tenant | None:\n # After TLS handshake, the client connection would use this method to\n # retrieve the Tenant instance associated with the given SSLObject.\n #\n # This method must not fail. See also _sni_callback() above.\n return self.get_default_tenant()\n\n def get_default_tenant(self) -> edbtenant.Tenant:\n # The client connection must proceed on a Tenant instance. In cases:\n # 1. plain-text connection without TLS handshake\n # 2. TLS handshake didn't provide SNI\n # 3. SNI didn't match any Tenant (retrieve_tenant() returned None)\n # this method will be called for a \"default\" tenant to use.\n #\n # The caller must be ready to handle errors raised in this method, and\n # provide a decent error.\n raise NotImplementedError\n\n def iter_tenants(self) -> Iterator[edbtenant.Tenant]:\n raise NotImplementedError\n\n async def maybe_generate_pki(\n self, args: srvargs.ServerConfig, ss: BaseServer\n ) -> tuple[bool, bool]:\n tls_cert_newly_generated = False\n if args.tls_cert_mode is srvargs.ServerTlsCertMode.SelfSigned:\n assert args.tls_cert_file is not None\n if not args.tls_cert_file.exists():\n assert args.tls_key_file is not None\n logger.info(\n f'generating self-signed TLS certificate '\n f'in \"{args.tls_cert_file}\"'\n )\n secretkey.generate_tls_cert(\n args.tls_cert_file,\n args.tls_key_file,\n ss.get_listen_hosts(),\n )\n tls_cert_newly_generated = True\n jws_keys_newly_generated = False\n if args.jose_key_mode is srvargs.JOSEKeyMode.Generate:\n assert args.jws_key_file is not None\n if not args.jws_key_file.exists():\n logger.info(\n f'generating JOSE key pair in \"{args.jws_key_file}\"'\n )\n auth.generate_jwk(args.jws_key_file)\n jws_keys_newly_generated = True\n return tls_cert_newly_generated, jws_keys_newly_generated\n\n\nclass Server(BaseServer):\n _tenant: edbtenant.Tenant\n _startup_script: srvargs.StartupScript | None\n _new_instance: bool\n\n def __init__(\n self,\n *,\n tenant: edbtenant.Tenant,\n startup_script: srvargs.StartupScript | None = None,\n new_instance: bool,\n **kwargs,\n ):\n super().__init__(**kwargs)\n self._tenant = tenant\n self._startup_script = startup_script\n self._new_instance = new_instance\n\n tenant.set_server(self)\n\n def _get_sys_config(self) -> Mapping[str, config.SettingValue]:\n return self._tenant.get_sys_config()\n\n async def init(self) -> None:\n logger.debug(\"starting server init\")\n await self._tenant.init_sys_pgcon()\n await self._load_instance_data()\n await self._maybe_patch()\n await self._tenant.init()\n await super().init()\n\n def get_default_tenant(self) -> edbtenant.Tenant:\n return self._tenant\n\n def iter_tenants(self) -> Iterator[edbtenant.Tenant]:\n yield self._tenant\n\n async def _get_patch_log(\n self, conn: pgcon.PGConnection, idx: int\n ) -> Optional[bootstrap.PatchEntry]:\n # We need to maintain a log in the system database of\n # patches that have been applied. This is so that if a\n # patch creates a new object, and then we succesfully\n # apply the patch to a user db but crash *before* applying\n # it to the system db, when we start up again and try\n # applying it to the system db, it is important that we\n # apply the same compiled version of the patch. If we\n # instead recompiled it, and it created new objects, those\n # objects might have a different id in the std schema and\n # in the actual user db.\n result = await instdata.get_instdata(\n conn, f'patch_log_{idx}', 'bin')\n if result:\n return pickle.loads(result)\n else:\n return None\n\n async def _prepare_patches(\n self, conn: pgcon.PGConnection\n ) -> dict[int, bootstrap.PatchEntry]:\n \"\"\"Prepare all the patches\"\"\"\n num_patches = await self._tenant.get_patch_count(conn)\n\n if num_patches < len(pg_patches.PATCHES):\n logger.info(\"preparing patches for database upgrade\")\n\n patches = {}\n patch_list = list(enumerate(pg_patches.PATCHES))\n for num, (kind, patch) in patch_list[num_patches:]:\n from . import bootstrap # noqa: F402\n\n idx = num_patches + num\n if not (entry := await self._get_patch_log(conn, idx)):\n patch_info = await bootstrap.gather_patch_info(\n num, kind, patch, conn\n )\n\n entry = bootstrap.prepare_patch(\n num, kind, patch, self._std_schema, self._refl_schema,\n self._schema_class_layout,\n self._tenant.get_backend_runtime_params(),\n patch_info=patch_info,\n )\n\n await bootstrap._store_static_bin_cache_conn(\n conn, f'patch_log_{idx}', pickle.dumps(entry))\n\n patches[num] = entry\n _, _, updates = entry\n if 'std_and_reflection_schema' in updates:\n self._std_schema, self._refl_schema = updates[\n 'std_and_reflection_schema']\n # +config patches might modify config_spec, which requires\n # a reload of it from the schema.\n if '+config' in kind:\n config_spec = config.load_spec_from_schema(self._std_schema)\n self._config_settings = config_spec\n\n if 'local_intro_query' in updates:\n self._local_intro_query = updates['local_intro_query']\n if 'global_intro_query' in updates:\n self._global_intro_query = updates['global_intro_query']\n if 'classlayout' in updates:\n self._schema_class_layout = updates['classlayout']\n if 'sysqueries' in updates:\n queries = json.loads(updates['sysqueries'])\n self._sys_queries = immutables.Map(\n {k: q.encode() for k, q in queries.items()})\n if 'report_configs_typedesc' in updates:\n self._report_config_typedesc = (\n updates['report_configs_typedesc'])\n\n return patches\n\n async def _maybe_apply_patches(\n self,\n dbname: str,\n conn: pgcon.PGConnection,\n patches: dict[int, bootstrap.PatchEntry],\n sys: bool=False,\n ) -> None:\n \"\"\"Apply any un-applied patches to the database.\"\"\"\n num_patches = await self._tenant.get_patch_count(conn)\n for num, (sql_b, syssql, keys) in patches.items():\n if num_patches <= num:\n if sys:\n sql_b += syssql\n logger.info(\"applying patch %d to database '%s'\", num, dbname)\n sql = tuple(x.encode('utf-8') for x in sql_b)\n\n # For certain things, we need to actually run it\n # against each user database.\n if keys.get('is_user_update'):\n from . import bootstrap\n\n kind, patch = pg_patches.PATCHES[num]\n patch_info = await bootstrap.gather_patch_info(\n num, kind, patch, conn\n )\n\n # Reload the compiler state from this database in\n # particular, so we can compiler from exactly the\n # right state. (Since self._std_schema and the like might\n # be further advanced.)\n state = (await edbcompiler.new_compiler_from_pg(conn)).state\n\n assert state.global_intro_query and state.local_intro_query\n global_schema = self._parse_global_schema(\n await conn.sql_fetch_val(\n state.global_intro_query.encode('utf-8')),\n )\n user_schema = self._parse_user_schema(\n await conn.sql_fetch_val(\n state.local_intro_query.encode('utf-8')),\n global_schema,\n )\n\n entry = bootstrap.prepare_patch(\n num, kind, patch,\n state.std_schema,\n state.refl_schema,\n state.schema_class_layout,\n self._tenant.get_backend_runtime_params(),\n patch_info=patch_info,\n user_schema=user_schema,\n global_schema=global_schema,\n dbname=dbname,\n )\n\n sql += tuple(x.encode('utf-8') for x in entry[0])\n\n if sql:\n await conn.sql_execute(sql)\n logger.info(\n \"finished applying patch %d to database '%s'\", num, dbname)\n\n async def _maybe_patch_db(\n self, dbname: str, patches: dict[int, bootstrap.PatchEntry], sem: Any\n ) -> None:\n logger.info(\"applying patches to database '%s'\", dbname)\n\n try:\n async with sem:\n async with self._tenant.direct_pgcon(dbname) as conn:\n await self._maybe_apply_patches(dbname, conn, patches)\n except Exception as e:\n if (\n isinstance(e, errors.EdgeDBError)\n and not isinstance(e, errors.InternalServerError)\n ):\n raise\n raise errors.InternalServerError(\n f'Could not apply patches for minor version upgrade to '\n f'database {dbname}'\n ) from e\n\n async def _maybe_patch(self) -> None:\n \"\"\"Apply patches to all the databases\"\"\"\n\n async with self._tenant.use_sys_pgcon() as syscon:\n patches = await self._prepare_patches(syscon)\n if not patches:\n return\n\n dbnames = await self.get_dbnames(syscon)\n\n async with asyncio.TaskGroup() as g:\n # Cap the parallelism used when applying patches, to avoid\n # having huge numbers of in flight patches that make\n # little visible progress in the logs.\n sem = asyncio.Semaphore(16)\n\n # Patch all the databases\n for dbname in dbnames:\n if dbname != defines.EDGEDB_SYSTEM_DB:\n g.create_task(\n self._maybe_patch_db(dbname, patches, sem))\n\n # Patch the template db, so that any newly created databases\n # will have the patches.\n g.create_task(self._maybe_patch_db(\n defines.EDGEDB_TEMPLATE_DB, patches, sem))\n\n await self._tenant.ensure_database_not_connected(\n defines.EDGEDB_TEMPLATE_DB\n )\n\n # Patch the system db last. The system db needs to go last so\n # that it only gets updated if all of the other databases have\n # been succesfully patched. This is important, since we don't check\n # other databases for patches unless the system db is patched.\n #\n # Driving everything from the system db like this lets us\n # always use the correct schema when compiling patches.\n async with self._tenant.use_sys_pgcon() as syscon:\n await self._maybe_apply_patches(\n defines.EDGEDB_SYSTEM_DB, syscon, patches, sys=True)\n\n def _load_schema(self, result, version_key) -> s_schema.Schema:\n res = pickle.loads(result[2:])\n if version_key != pg_patches.get_version_key(len(pg_patches.PATCHES)):\n res = s_schema.upgrade_schema(res)\n return res\n\n async def _load_instance_data(self):\n logger.info(\"loading instance data\")\n async with self._tenant.use_sys_pgcon() as syscon:\n patch_count = await self._tenant.get_patch_count(syscon)\n version_key = pg_patches.get_version_key(patch_count)\n\n result = await instdata.get_instdata(\n syscon, f'sysqueries{version_key}', 'json')\n queries = json.loads(result)\n self._sys_queries = immutables.Map(\n {k: q.encode() for k, q in queries.items()})\n\n self._report_config_typedesc[(1, 0)] = (\n await instdata.get_instdata(\n syscon,\n f'report_configs_typedesc_1_0{version_key}',\n 'bin',\n )\n )\n\n self._report_config_typedesc[(2, 0)] = (\n await instdata.get_instdata(\n syscon,\n f'report_configs_typedesc_2_0{version_key}',\n 'bin',\n )\n )\n\n def _reload_stmt_cache_size(self):\n size = self.config_lookup(\n '_pg_prepared_statement_cache_size', self._get_sys_config()\n )\n self._stmt_cache_size = size\n self._tenant.set_stmt_cache_size(size)\n\n async def _restart_servers_new_addr(self, nethosts, netport):\n if not netport:\n raise RuntimeError('cannot restart without network port specified')\n nethosts, has_ipv4_wc, has_ipv6_wc = await _resolve_interfaces(\n nethosts\n )\n servers_to_stop = []\n servers_to_stop_early = []\n servers = {}\n if self._listen_port == netport:\n hosts_to_start = [\n host for host in nethosts if host not in self._servers\n ]\n for host, srv in self._servers.items():\n if host == ADMIN_PLACEHOLDER or host in nethosts:\n servers[host] = srv\n elif host in ['::', '0.0.0.0']:\n servers_to_stop_early.append(srv)\n else:\n if has_ipv4_wc:\n try:\n ipaddress.IPv4Address(host)\n except ValueError:\n pass\n else:\n servers_to_stop_early.append(srv)\n continue\n if has_ipv6_wc:\n try:\n ipaddress.IPv6Address(host)\n except ValueError:\n pass\n else:\n servers_to_stop_early.append(srv)\n continue\n servers_to_stop.append(srv)\n admin = False\n else:\n hosts_to_start = nethosts\n servers_to_stop = list(self._servers.values())\n admin = True\n\n if servers_to_stop_early:\n await self._stop_servers_with_logging(servers_to_stop_early)\n\n if hosts_to_start:\n try:\n new_servers, *_ = await self._start_servers(\n tuple(hosts_to_start),\n netport,\n admin=admin,\n )\n servers.update(new_servers)\n except StartupError:\n raise errors.ConfigurationError(\n 'Server updated its config but cannot serve on requested '\n 'address/port, please see server log for more information.'\n )\n self._servers = servers\n self._listen_hosts = [\n s.getsockname()[0]\n for host, tcp_srv in servers.items()\n if host != ADMIN_PLACEHOLDER\n for s in tcp_srv.sockets # type: ignore\n ]\n self._listen_port = netport\n\n await self._stop_servers_with_logging(servers_to_stop)\n\n async def _stop_servers_with_logging(self, servers_to_stop):\n addrs = []\n unix_addr = None\n port = None\n for srv in servers_to_stop:\n for s in srv.sockets:\n addr = s.getsockname()\n if isinstance(addr, tuple):\n addrs.append(addr[:2])\n if port is None:\n port = addr[1]\n elif port != addr[1]:\n port = 0\n else:\n unix_addr = addr\n if len(addrs) > 1:\n if port:\n addr_str = f\"{{{', '.join(addr[0] for addr in addrs)}}}:{port}\"\n else:\n addr_str = f\"{{{', '.join('%s:%d' % addr for addr in addrs)}}}\"\n elif addrs:\n addr_str = \"%s:%d\" % addrs[0]\n else:\n addr_str = None\n if addr_str:\n logger.info('Stopping to serve on %s', addr_str)\n if unix_addr:\n logger.info('Stopping to serve admin on %s', unix_addr)\n\n await self._stop_servers(servers_to_stop)\n\n async def _on_system_config_set(self, setting_name, value):\n try:\n if setting_name == 'listen_addresses':\n await self._restart_servers_new_addr(value, self._listen_port)\n\n elif setting_name == 'listen_port':\n await self._restart_servers_new_addr(self._listen_hosts, value)\n\n elif setting_name == 'session_idle_timeout':\n self.reinit_idle_gc_collector()\n\n elif setting_name == '_pg_prepared_statement_cache_size':\n self._reload_stmt_cache_size()\n\n self._tenant.schedule_reported_config_if_needed(setting_name)\n except Exception:\n metrics.background_errors.inc(\n 1.0, self._tenant.get_instance_name(), 'on_system_config_set'\n )\n raise\n\n async def _on_system_config_reset(self, setting_name):\n try:\n if setting_name == 'listen_addresses':\n cfg = self._get_sys_config()\n await self._restart_servers_new_addr(\n self.config_lookup('listen_addresses', cfg)\n or ('localhost',),\n self._listen_port,\n )\n\n elif setting_name == 'listen_port':\n cfg = self._get_sys_config()\n await self._restart_servers_new_addr(\n self._listen_hosts,\n self.config_lookup('listen_port', cfg)\n or defines.EDGEDB_PORT,\n )\n\n elif setting_name == 'session_idle_timeout':\n self.reinit_idle_gc_collector()\n\n elif setting_name == '_pg_prepared_statement_cache_size':\n self._reload_stmt_cache_size()\n\n self._tenant.schedule_reported_config_if_needed(setting_name)\n except Exception:\n metrics.background_errors.inc(\n 1.0, self._tenant.get_instance_name(), 'on_system_config_reset'\n )\n raise\n\n async def _after_system_config_add(self, setting_name, value):\n try:\n if setting_name == 'auth':\n self._tenant.populate_sys_auth()\n except Exception:\n metrics.background_errors.inc(\n 1.0,\n self._tenant.get_instance_name(),\n 'after_system_config_add',\n )\n raise\n\n async def _after_system_config_rem(self, setting_name, value):\n try:\n if setting_name == 'auth':\n self._tenant.populate_sys_auth()\n except Exception:\n metrics.background_errors.inc(\n 1.0,\n self._tenant.get_instance_name(),\n 'after_system_config_rem',\n )\n raise\n\n async def run_startup_script_and_exit(self):\n \"\"\"Run the script specified in *startup_script* and exit immediately\"\"\"\n if self._startup_script is None:\n raise AssertionError('startup script is not defined')\n pool = await compiler_pool.create_compiler_pool(\n **self._get_compiler_args()\n )\n self.compilation_config_serializer = (\n await pool.make_compilation_config_serializer()\n )\n self._compiler_pool = pool\n try:\n await binary.run_script(\n server=self,\n tenant=self._tenant,\n database=self._startup_script.database,\n user=self._startup_script.user,\n script=self._startup_script.text,\n )\n finally:\n await self._destroy_compiler_pool()\n\n async def _before_start_servers(self) -> None:\n await self._tenant.start_accepting_new_tasks()\n if self._startup_script and self._new_instance:\n await binary.run_script(\n server=self,\n tenant=self._tenant,\n database=self._startup_script.database,\n user=self._startup_script.user,\n script=self._startup_script.text,\n )\n\n async def _after_start_servers(self) -> None:\n self._tenant.start_running()\n\n def _get_status(self) -> dict[str, Any]:\n status = super()._get_status()\n status[\"tenant_id\"] = self._tenant.tenant_id\n return status\n\n def load_jwcrypto(self, jws_key_file: pathlib.Path) -> auth.JWKSet:\n jws_key = super().load_jwcrypto(jws_key_file)\n self._tenant.load_jwcrypto(jws_key)\n return jws_key\n\n def request_shutdown(self):\n self._tenant.stop_accepting_connections()\n super().request_shutdown()\n\n async def stop(self):\n try:\n self._tenant.stop()\n\n await super().stop()\n\n await self._tenant.wait_stopped()\n await self._destroy_compiler_pool()\n finally:\n self._tenant.terminate_sys_pgcon()\n\n def get_debug_info(self):\n parent = super().get_debug_info()\n child = self._tenant.get_debug_info()\n parent[\"params\"].update(child[\"params\"])\n child[\"params\"] = parent[\"params\"]\n parent.update(child)\n return parent\n\n def _get_backend_runtime_params(self) -> pgparams.BackendRuntimeParams:\n return self._tenant.get_backend_runtime_params()\n\n def _get_compiler_args(self) -> dict[str, Any]:\n rv = super()._get_compiler_args()\n rv.update(self._tenant.get_compiler_args())\n return rv\n\n def start_watching_files(self):\n super().start_watching_files()\n self._tenant.start_watching_files()\n\n\ndef _cleanup_wildcard_addrs(\n hosts: Sequence[str],\n) -> tuple[list[str], list[str], bool, bool]:\n \"\"\"Filter out conflicting addresses in presence of INADDR_ANY wildcards.\n\n Attempting to bind to 0.0.0.0 (or ::) _and_ a non-wildcard address will\n usually result in EADDRINUSE. To avoid this, filter out all specific\n addresses if a wildcard is present in the *hosts* sequence.\n\n Returns a tuple: first element is the new list of hosts, second\n element is a list of rejected host addrs/names.\n \"\"\"\n\n ipv4_hosts = set()\n ipv6_hosts = set()\n named_hosts = set()\n\n ipv4_wc = ipaddress.ip_address('0.0.0.0')\n ipv6_wc = ipaddress.ip_address('::')\n\n for host in hosts:\n if host == \"*\":\n ipv4_hosts.add(ipv4_wc)\n ipv6_hosts.add(ipv6_wc)\n continue\n\n try:\n ip = ipaddress.IPv4Address(host)\n except ValueError:\n pass\n else:\n ipv4_hosts.add(ip)\n continue\n\n try:\n ip6 = ipaddress.IPv6Address(host)\n except ValueError:\n pass\n else:\n ipv6_hosts.add(ip6)\n continue\n\n named_hosts.add(host)\n\n if not ipv4_hosts and not ipv6_hosts:\n return (list(hosts), [], False, False)\n\n if ipv4_wc not in ipv4_hosts and ipv6_wc not in ipv6_hosts:\n return (list(hosts), [], False, False)\n\n if ipv4_wc in ipv4_hosts and ipv6_wc in ipv6_hosts:\n return (\n ['0.0.0.0', '::'],\n [\n str(a) for a in\n ((named_hosts | ipv4_hosts | ipv6_hosts) - {ipv4_wc, ipv6_wc})\n ],\n True,\n True,\n )\n\n if ipv4_wc in ipv4_hosts:\n return (\n [str(a) for a in ({ipv4_wc} | ipv6_hosts)],\n [str(a) for a in ((named_hosts | ipv4_hosts) - {ipv4_wc})],\n True,\n False,\n )\n\n if ipv6_wc in ipv6_hosts:\n return (\n [str(a) for a in ({ipv6_wc} | ipv4_hosts)],\n [str(a) for a in ((named_hosts | ipv6_hosts) - {ipv6_wc})],\n False,\n True,\n )\n\n raise AssertionError('unreachable')\n\n\nasync def _resolve_host(host: str) -> list[str] | Exception:\n loop = asyncio.get_running_loop()\n try:\n addrinfo = await loop.getaddrinfo(\n None if host == '*' else host,\n 0,\n family=socket.AF_UNSPEC,\n type=socket.SOCK_STREAM,\n flags=socket.AI_PASSIVE,\n )\n except Exception as e:\n return e\n else:\n return [addr[4][0] for addr in addrinfo]\n\n\nasync def _resolve_interfaces(\n hosts: Sequence[str],\n) -> tuple[Sequence[str], bool, bool]:\n\n async with asyncio.TaskGroup() as g:\n resolve_tasks = {\n host: g.create_task(_resolve_host(host))\n for host in hosts\n }\n\n addrs = []\n for host, fut in resolve_tasks.items():\n result = fut.result()\n if isinstance(result, Exception):\n logger.warning(\n f\"could not translate host name {host!r} to address: {result}\")\n else:\n addrs.extend(result)\n\n (\n clean_addrs, rejected_addrs, has_ipv4_wc, has_ipv6_wc\n ) = _cleanup_wildcard_addrs(addrs)\n\n if rejected_addrs:\n logger.warning(\n \"wildcard addresses found in listen_addresses; \" +\n \"discarding the other addresses: \" +\n \", \".join(repr(h) for h in rejected_addrs)\n )\n\n return clean_addrs, has_ipv4_wc, has_ipv6_wc\n" }, { "path": "edb/server/service_manager.py", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2016-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\nfrom __future__ import annotations\nfrom typing import Optional\n\nimport errno\nimport logging\nimport os\nimport socket\nimport sys\n\n\nSD_LISTEN_FDS_START = 3\n\n\nlogger = logging.getLogger('edb.server')\n\n\ndef _stream_socket_from_fd(fd: int) -> Optional[socket.socket]:\n try:\n sock = socket.socket(fileno=fd)\n except OSError:\n logger.warning(\n f\"activation file descriptor {fd} is not a socket \"\n f\", ignoring\"\n )\n return None\n\n if sock.family not in {socket.AF_INET, socket.AF_INET6}:\n logger.warning(\n f\"activation file descriptor {fd} is not an AF_INET[6] socket \"\n f\", ignoring\"\n )\n return None\n\n if sock.type != socket.SOCK_STREAM:\n logger.warning(\n f\"activation file descriptor {fd} is not an SOCK_STREAM \"\n f\"socket, ignoring\"\n )\n return None\n\n return sock\n\n\ndef sd_notify(message: str) -> None:\n notify_socket = os.environ.get('NOTIFY_SOCKET')\n if not notify_socket:\n return\n\n if notify_socket[0] == '@':\n notify_socket = '\\0' + notify_socket[1:]\n\n with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sd_sock:\n try:\n sd_sock.connect(notify_socket)\n sd_sock.sendall(message.encode())\n except Exception as e:\n logger.info('Could not send systemd notification: %s', e)\n\n\ndef sd_get_activation_listen_sockets() -> dict[str, list[socket.socket]]:\n # Prevent socket activation variables from being inherited by\n # child processes (regardless of success below).\n listen_pid = os.environ.pop(\"LISTEN_PID\", \"\")\n listen_fds = os.environ.pop(\"LISTEN_FDS\", \"\")\n listen_fdnames = os.environ.pop(\"LISTEN_FDNAMES\", \"\")\n\n if not listen_pid or not listen_fds:\n return {}\n\n try:\n expected_pid = int(listen_pid)\n except ValueError:\n logger.warning(\n \"the value of LISTEN_PID environment variable \"\n \"is not a valid integer, ignoring socket activation data\"\n )\n return {}\n\n if expected_pid != os.getpid():\n logger.warning(\n \"the value of LISTEN_PID does not match the PID of this \"\n \"process, ignoring socket activation data\"\n )\n return {}\n\n try:\n num_fds = int(listen_fds)\n except ValueError:\n logger.warning(\n \"the value of LISTEN_FDS environment variable \"\n \"is not a valid integer, ignoring socket activation data\"\n )\n return {}\n\n fd_names = listen_fdnames.split(\":\")\n fd_range = range(SD_LISTEN_FDS_START, SD_LISTEN_FDS_START + num_fds)\n sockets: dict[str, list[socket.socket]] = {}\n\n for i, fd in enumerate(fd_range):\n os.set_inheritable(fd, False)\n\n try:\n name = fd_names[i]\n except IndexError:\n name = f\"LISTEN_FD_{fd}\"\n\n sock = _stream_socket_from_fd(fd)\n if sock is not None:\n sockets.setdefault(name, []).append(sock)\n\n return sockets\n\n\nif sys.platform == \"darwin\":\n import ctypes\n import ctypes.util\n\n syslib = ctypes.CDLL(ctypes.util.find_library('System'))\n syslib.launch_activate_socket.argypes = [ # type: ignore[attr-defined]\n ctypes.c_char_p,\n ctypes.POINTER(ctypes.POINTER(ctypes.c_int)),\n ctypes.POINTER(ctypes.c_size_t),\n ]\n\n class LaunchActivateSocketError(Exception):\n def __init__(self, errno: int) -> None:\n self.errno = errno\n\n def _launch_activate_socket(name) -> list[int]:\n fds = ctypes.POINTER(ctypes.c_int)()\n num_fds = ctypes.c_size_t()\n result = syslib.launch_activate_socket(\n ctypes.c_char_p(name.encode(\"utf-8\")),\n ctypes.byref(fds),\n ctypes.byref(num_fds),\n )\n if result == 0:\n return [fds[i] for i in range(num_fds.value)]\n elif result == errno.ESRCH:\n # Not running under launchd\n return []\n else:\n raise LaunchActivateSocketError(result)\n\n def launchd_get_activation_listen_sockets() -> (\n dict[str, list[socket.socket]]\n ):\n names = [\"edgedb-server\"]\n sockets: dict[str, list[socket.socket]] = {}\n\n for name in names:\n try:\n fds = _launch_activate_socket(name)\n except LaunchActivateSocketError as e:\n logger.warning(\n f\"could not activate socket {name}: \"\n f\"launch_activate_socket() returned {e.errno}\")\n continue\n\n for fd in fds:\n os.set_inheritable(fd, False)\n sock = _stream_socket_from_fd(fd)\n if sock is not None:\n sockets.setdefault(name, []).append(sock)\n\n return sockets\n\nelse:\n\n def launchd_get_activation_listen_sockets() -> (\n dict[str, list[socket.socket]]\n ):\n return {}\n\n\ndef get_activation_listen_sockets() -> dict[str, list[socket.socket]]:\n if sys.platform == \"darwin\":\n sockets = launchd_get_activation_listen_sockets()\n else:\n sockets = sd_get_activation_listen_sockets()\n\n return sockets\n" }, { "path": "edb/server/smtp.py", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2024-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\nfrom __future__ import annotations\n\nimport dataclasses\nimport email.message\nimport asyncio\nimport logging\nimport os\nimport hashlib\nimport pickle\nimport aiosmtplib\n\nfrom typing import Optional\n\nfrom edb.common import retryloop\nfrom edb.ir import statypes\nfrom edb import errors\nfrom . import dbview\n\n\n_semaphore: asyncio.BoundedSemaphore | None = None\n\nlogger = logging.getLogger('edb.server.smtp')\n\n\n@dataclasses.dataclass\nclass SMTPProviderConfig:\n name: str\n sender: Optional[str]\n host: Optional[str]\n port: Optional[int]\n username: Optional[str]\n password: Optional[str]\n security: str\n validate_certs: bool\n timeout_per_email: statypes.Duration\n timeout_per_attempt: statypes.Duration\n\n\nclass SMTP:\n def __init__(self, db: dbview.Database):\n current_provider = get_current_email_provider(db)\n self.sender = current_provider.sender or \"noreply@example.com\"\n default_port = (\n 465\n if current_provider.security == \"TLS\"\n else 587 if current_provider.security == \"STARTTLS\" else 25\n )\n use_tls: bool\n start_tls: bool | None\n match current_provider.security:\n case \"PlainText\":\n use_tls = False\n start_tls = False\n\n case \"TLS\":\n use_tls = True\n start_tls = False\n\n case \"STARTTLS\":\n use_tls = False\n start_tls = True\n\n case \"STARTTLSOrPlainText\":\n use_tls = False\n start_tls = None\n\n case _:\n raise NotImplementedError\n\n host = current_provider.host or \"localhost\"\n port = current_provider.port or default_port\n username = current_provider.username\n password = current_provider.password\n validate_certs = current_provider.validate_certs\n timeout_per_attempt = current_provider.timeout_per_attempt\n\n req_timeout = timeout_per_attempt.to_microseconds() / 1_000_000.0\n self.timeout_per_email = (\n current_provider.timeout_per_email.to_microseconds() / 1_000_000.0\n )\n self.client = aiosmtplib.SMTP(\n hostname=host,\n port=port,\n username=username,\n password=password,\n timeout=req_timeout,\n use_tls=use_tls,\n start_tls=start_tls,\n validate_certs=validate_certs,\n )\n\n async def send(\n self,\n message: email.message.Message,\n *,\n test_mode: bool = False,\n ) -> None:\n global _semaphore\n if _semaphore is None:\n _semaphore = asyncio.BoundedSemaphore(\n int(\n os.environ.get(\n \"EDGEDB_SERVER_AUTH_SMTP_CONCURRENCY\",\n os.environ.get(\"EDGEDB_SERVER_SMTP_CONCURRENCY\", 5),\n )\n )\n )\n\n # n.b. When constructing EmailMessage objects, we don't set the \"From\"\n # header since that is configured in the SmtpProviderConfig. However,\n # the EmailMessage will have the correct \"To\" header.\n message[\"From\"] = self.sender\n rloop = retryloop.RetryLoop(\n timeout=self.timeout_per_email,\n backoff=retryloop.exp_backoff(),\n ignore=(\n aiosmtplib.SMTPConnectError,\n aiosmtplib.SMTPHeloError,\n aiosmtplib.SMTPServerDisconnected,\n aiosmtplib.SMTPConnectTimeoutError,\n aiosmtplib.SMTPConnectResponseError,\n ),\n )\n async for iteration in rloop:\n async with iteration:\n async with _semaphore:\n # Currently we are not reusing SMTP connections, but\n # ideally we should replace this with a pool of\n # connections, and drop idle connections after configured\n # time.\n if test_mode:\n self._send_test_mode_email(message)\n else:\n logger.info(\n \"Sending SMTP message to \"\n f\"{self.client.hostname}:{self.client.port}\"\n )\n\n async with self.client:\n errors, response = await self.client.send_message(\n message\n )\n if errors:\n logger.error(\n f\"SMTP server returned errors: {errors}\"\n )\n else:\n logger.info(\n f\"SMTP message sent successfully: {response}\"\n )\n\n def _send_test_mode_email(self, message: email.message.Message):\n sender = message[\"From\"]\n recipients = message[\"To\"]\n recipients_list: list[str]\n if isinstance(recipients, str):\n recipients_list = [recipients]\n elif recipients is None:\n recipients_list = []\n else:\n recipients_list = list(recipients)\n\n hash_input = f\"{sender}{','.join(recipients_list)}\"\n file_name_hash = hashlib.sha256(hash_input.encode()).hexdigest()\n file_name = f\"/tmp/edb-test-email-{file_name_hash}.pickle\"\n test_file = os.environ.get(\n \"EDGEDB_TEST_EMAIL_FILE\",\n file_name,\n )\n if os.path.exists(test_file):\n os.unlink(test_file)\n with open(test_file, \"wb\") as f:\n logger.info(f\"Dumping SMTP message to {test_file}\")\n args = dict(\n message=message,\n sender=sender,\n recipients=recipients,\n hostname=self.client.hostname,\n port=self.client.port,\n username=self.client._login_username,\n password=self.client._login_password,\n timeout=self.client.timeout,\n use_tls=self.client.use_tls,\n start_tls=self.client._start_tls_on_connect,\n validate_certs=self.client.validate_certs,\n )\n pickle.dump(args, f)\n\n\ndef get_current_email_provider(\n db: dbview.Database,\n) -> SMTPProviderConfig:\n current_provider_name = db.lookup_config(\"current_email_provider_name\")\n if current_provider_name is None:\n raise errors.ConfigurationError(\"No email provider configured\")\n\n found = None\n objs = (\n list(db.lookup_config(\"email_providers\"))\n + db.tenant._sidechannel_email_configs\n )\n for obj in objs:\n if obj.name == current_provider_name:\n values = {}\n for field in dataclasses.fields(SMTPProviderConfig):\n key = field.name\n values[key] = getattr(obj, key)\n found = SMTPProviderConfig(**values)\n break\n\n if found is None:\n raise errors.ConfigurationError(\n f\"No email provider named {current_provider_name!r}\"\n )\n return found\n" }, { "path": "edb/server/tenant.py", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2023-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\nfrom __future__ import annotations\nfrom typing import (\n Any,\n Callable,\n Iterator,\n Iterable,\n Mapping,\n Coroutine,\n AsyncGenerator,\n Optional,\n TypedDict,\n TYPE_CHECKING,\n)\n\nimport asyncio\nimport contextlib\nimport dataclasses\nimport json\nimport logging\nimport os\nimport pathlib\nimport pickle\nimport struct\nimport sys\nimport textwrap\nimport time\nimport tomllib\nimport uuid\nimport weakref\n\nimport immutables\n\nfrom edb import buildmeta\nfrom edb import errors\nfrom edb.common import asyncutil\nfrom edb.common import lru\nfrom edb.common import retryloop\nfrom edb.common import verutils\nfrom edb.common.log import current_tenant\n\nfrom . import auth\nfrom . import args as srvargs\nfrom . import config\nfrom . import connpool\nfrom . import dbview\nfrom . import defines\nfrom . import instdata\nfrom . import metrics\nfrom . import pgcon\nfrom . import compiler as edbcompiler\nfrom . import pgconnparams\n\nfrom .ha import adaptive as adaptive_ha\nfrom .ha import base as ha_base\nfrom .http import HttpClient\nfrom .pgcon import errors as pgcon_errors\nfrom .compiler import enums as compiler_enums\n\nif TYPE_CHECKING:\n from edb.pgsql import params as pgparams\n\n from . import pgcluster\n from . import server as edbserver\n from . import compiler_pool as edbcompiler_pool\n\n\nlogger = logging.getLogger(\"edb.server\")\n\n\nHTTP_MAX_CONNECTIONS = 100\nHEALTH_CHECK_MIN_INTERVAL: float = float(\n os.getenv(\"GEL_BACKEND_HEALTH_CHECK_MIN_INTERVAL\", 10)\n)\nHEALTH_CHECK_TIMEOUT: float = float(\n os.getenv(\"GEL_BACKEND_HEALTH_CHECK_TIMEOUT\", 10)\n)\n\n\nclass RoleDescriptor(TypedDict):\n superuser: bool\n name: str\n password: str | None\n all_permissions: list[str] | None\n branches: list[str]\n apply_access_policies_pg_default: bool | None\n\n\nclass Tenant(ha_base.ClusterProtocol):\n _server: edbserver.BaseServer\n _cluster: pgcluster.BaseCluster\n _tenant_id: str\n _instance_name: str\n _instance_data: Mapping[str, str]\n _dbindex: dbview.DatabaseIndex | None\n _initing: bool\n _running: bool\n _accepting_connections: bool\n _introspection_locks: weakref.WeakValueDictionary[str, asyncio.Lock]\n\n __loop: asyncio.AbstractEventLoop\n _task_group: asyncio.TaskGroup | None\n _tasks: set[asyncio.Task]\n _accept_new_tasks: bool\n _file_watch_finalizers: list[Callable[[], None]]\n\n __sys_pgcon: pgcon.PGConnection | None\n _sys_pgcon_waiter: asyncio.Lock\n _sys_pgcon_ready_evt: asyncio.Event\n _sys_pgcon_reconnect_evt: asyncio.Event\n _sys_pgcon_last_active_time: float\n _max_backend_connections: int\n _suggested_client_pool_size: int\n _pg_pool: connpool.Pool\n _pg_unavailable_msg: str | None\n _init_con_data: list[config.ConState]\n _init_con_sql: bytes | None\n\n _ha_master_serial: int\n _backend_adaptive_ha: adaptive_ha.AdaptiveHASupport | None\n _readiness_state_file: pathlib.Path | None\n _readiness: srvargs.ReadinessState\n _readiness_reason: str\n _config_file: pathlib.Path | None\n\n _extensions_dirs: tuple[pathlib.Path, ...]\n\n # A set of databases that should not accept new connections.\n _block_new_connections: set[str]\n _report_config_data: dict[defines.ProtocolVersion, bytes]\n\n _roles: Mapping[str, RoleDescriptor]\n _role_capabilities: Mapping[str, compiler_enums.Capability]\n _sys_auth: tuple[Any, ...]\n _jwt_sub_allowlist_file: pathlib.Path | None\n _jwt_sub_allowlist: frozenset[str] | None\n _jwt_revocation_list_file: pathlib.Path | None\n _jwt_revocation_list: frozenset[str] | None\n\n _http_client: HttpClient | None\n\n _sidechannel_email_configs: list[Any]\n\n def __init__(\n self,\n cluster: pgcluster.BaseCluster,\n *,\n instance_name: str,\n max_backend_connections: int,\n backend_adaptive_ha: bool = False,\n extensions_dir: tuple[pathlib.Path, ...] = (),\n ):\n self._cluster = cluster\n self._tenant_id = self.get_backend_runtime_params().tenant_id\n self._instance_name = instance_name\n self._instance_data = immutables.Map()\n self._initing = True\n self._running = False\n self._accepting_connections = False\n\n self._task_group = None\n self._tasks = set()\n self._named_tasks: dict[str, asyncio.Task] = dict()\n self._accept_new_tasks = False\n self._file_watch_finalizers = []\n self._introspection_locks = weakref.WeakValueDictionary()\n self._sidechannel_email_configs = []\n\n self._extensions_dirs = extensions_dir\n\n # Never use `self.__sys_pgcon` directly; get it via\n # `async with self.use_sys_pgcon()`.\n self.__sys_pgcon = None\n self._sys_pgcon_last_active_time = 0\n\n # Increase-only counter to reject outdated attempts to connect\n self._ha_master_serial = 0\n if backend_adaptive_ha:\n self._backend_adaptive_ha = adaptive_ha.AdaptiveHASupport(\n self, self._instance_name\n )\n else:\n self._backend_adaptive_ha = None\n self._readiness_state_file = None\n self._readiness = srvargs.ReadinessState.Default\n self._readiness_reason = \"\"\n self._config_file = None\n\n self._max_backend_connections = max_backend_connections\n self._suggested_client_pool_size = max(\n min(\n max_backend_connections, defines.MAX_SUGGESTED_CLIENT_POOL_SIZE\n ),\n defines.MIN_SUGGESTED_CLIENT_POOL_SIZE,\n )\n self._pg_pool = connpool.Pool(\n connect=self._pg_connect,\n disconnect=self._pg_disconnect,\n # 1 connection is reserved for the system DB\n max_capacity=max_backend_connections - 1,\n )\n self._pg_unavailable_msg = None\n self._block_new_connections = set()\n self._report_config_data = {}\n self._init_con_data = []\n self._init_con_sql = None\n\n # DB state will be initialized in init().\n self._dbindex = None\n\n self._branch_sem = asyncio.Semaphore(value=1)\n\n self._roles = immutables.Map()\n self._role_capabilities = immutables.Map()\n self._sys_auth = tuple()\n self._jwt_sub_allowlist_file = None\n self._jwt_sub_allowlist = None\n self._jwt_revocation_list_file = None\n self._jwt_revocation_list = None\n\n self._http_client = None\n\n # If it isn't stored in instdata, it is the old default.\n self.default_database = defines.EDGEDB_OLD_DEFAULT_DB\n\n def set_reloadable_files(\n self,\n readiness_state_file: str | pathlib.Path | None = None,\n jwt_sub_allowlist_file: str | pathlib.Path | None = None,\n jwt_revocation_list_file: str | pathlib.Path | None = None,\n config_file: str | pathlib.Path | None = None,\n ) -> bool:\n rv = False\n\n if isinstance(readiness_state_file, str):\n readiness_state_file = pathlib.Path(readiness_state_file)\n if self._readiness_state_file != readiness_state_file:\n self._readiness_state_file = readiness_state_file\n rv = True\n\n if isinstance(jwt_sub_allowlist_file, str):\n jwt_sub_allowlist_file = pathlib.Path(jwt_sub_allowlist_file)\n if self._jwt_sub_allowlist_file != jwt_sub_allowlist_file:\n self._jwt_sub_allowlist_file = jwt_sub_allowlist_file\n rv = True\n\n if isinstance(jwt_revocation_list_file, str):\n jwt_revocation_list_file = pathlib.Path(jwt_revocation_list_file)\n if self._jwt_revocation_list_file != jwt_revocation_list_file:\n self._jwt_revocation_list_file = jwt_revocation_list_file\n rv = True\n\n if isinstance(config_file, str):\n config_file = pathlib.Path(config_file)\n if self._config_file != config_file:\n self._config_file = config_file\n rv = True\n\n return rv\n\n def set_server(self, server: edbserver.BaseServer) -> None:\n self._server = server\n self.__loop = server.get_loop()\n\n async def load_sidechannel_configs(\n self,\n value: Any,\n *,\n compiler: (\n edbcompiler.Compiler | edbcompiler_pool.AbstractPool | None\n ) = None,\n ) -> None:\n if compiler is None:\n compiler = self._server.get_compiler_pool()\n objects = {\"cfg::Config\": {\"email_providers\": value}}\n if isinstance(compiler, edbcompiler.Compiler):\n result = compiler.compile_structured_config(\n objects, source=\"magic\", allow_nested=True\n )\n else:\n result = await compiler.compile_structured_config(\n objects, source=\"magic\", allow_nested=True\n )\n email_providers = result[\"cfg::Config\"][\"email_providers\"]\n self._sidechannel_email_configs = list(email_providers.value)\n\n def get_http_client(self, *, originator: str) -> HttpClient:\n if self._http_client is None:\n http_max_connections = self._server.config_lookup(\n 'http_max_connections', self.get_sys_config()\n )\n self._http_client = HttpClient(\n http_max_connections,\n user_agent=f\"EdgeDB {buildmeta.get_version_string(short=True)}\",\n stat_callback=lambda stat: logger.debug(\n f\"HTTP stat: {originator} {stat}\"\n )\n )\n return self._http_client\n\n def on_switch_over(self):\n # Bumping this serial counter will \"cancel\" all pending connections\n # to the old master.\n self._ha_master_serial += 1\n\n if self._accept_new_tasks:\n self.create_task(\n self._pg_pool.prune_all_connections(),\n interruptable=True,\n )\n\n if self.__sys_pgcon is None:\n # Assume a reconnect task is already running, now that we know the\n # new master is likely ready, let's just give the task a push.\n self._sys_pgcon_reconnect_evt.set()\n else:\n # Brutally close the sys_pgcon to the old master - this should\n # trigger a reconnect task.\n self.__sys_pgcon.abort()\n\n if self._backend_adaptive_ha is not None:\n # Switch to FAILOVER if adaptive HA is enabled\n self._backend_adaptive_ha.set_state_failover(\n call_on_switch_over=False\n )\n\n def get_active_pgcon_num(self) -> int:\n return self._pg_pool.active_conns\n\n @property\n def client_id(self) -> int:\n return self._cluster.get_client_id()\n\n @property\n def server(self) -> edbserver.BaseServer:\n return self._server\n\n @property\n def tenant_id(self) -> str:\n return self._tenant_id\n\n @property\n def suggested_client_pool_size(self) -> int:\n return self._suggested_client_pool_size\n\n def get_pg_dbname(self, dbname: str) -> str:\n return self._cluster.get_db_name(dbname)\n\n def get_pgaddr(self) -> pgconnparams.ConnectionParams:\n return self._cluster.get_pgaddr()\n\n @lru.method_cache\n def get_backend_runtime_params(self) -> pgparams.BackendRuntimeParams:\n return self._cluster.get_runtime_params()\n\n def get_instance_name(self) -> str:\n return self._instance_name\n\n def get_instance_data(self, key: str) -> str:\n return self._instance_data[key]\n\n def is_online(self) -> bool:\n return self._readiness is not srvargs.ReadinessState.Offline\n\n def is_blocked(self) -> bool:\n return self._readiness is srvargs.ReadinessState.Blocked\n\n def is_ready(self) -> bool:\n return (\n self._readiness is srvargs.ReadinessState.Default\n or self._readiness is srvargs.ReadinessState.ReadOnly\n )\n\n def is_readonly(self) -> bool:\n return self._readiness is srvargs.ReadinessState.ReadOnly\n\n def get_readiness_reason(self) -> str:\n return self._readiness_reason\n\n def get_sys_config(self) -> Mapping[str, config.SettingValue]:\n assert self._dbindex is not None\n return self._dbindex.get_sys_config()\n\n def get_report_config_data(\n self,\n protocol_version: defines.ProtocolVersion,\n ) -> bytes:\n if protocol_version >= (2, 0):\n return self._report_config_data[(2, 0)]\n else:\n return self._report_config_data[(1, 0)]\n\n def get_global_schema_pickle(self) -> bytes:\n assert self._dbindex is not None\n return self._dbindex.get_global_schema_pickle()\n\n def get_db(self, *, dbname: str) -> dbview.Database:\n assert self._dbindex is not None\n return self._dbindex.get_db(dbname)\n\n def maybe_get_db(self, *, dbname: str) -> dbview.Database | None:\n assert self._dbindex is not None\n return self._dbindex.maybe_get_db(dbname)\n\n def is_accepting_connections(self) -> bool:\n return self._accepting_connections and self._accept_new_tasks\n\n def get_roles(self) -> Mapping[str, RoleDescriptor]:\n return self._roles\n\n def set_roles(self, roles: Mapping[str, RoleDescriptor]) -> None:\n self._roles = roles\n self._refresh_role_capabilities()\n\n def get_role_capabilities(self) -> Mapping[str, compiler_enums.Capability]:\n return self._role_capabilities\n\n def _refresh_role_capabilities(self) -> None:\n role_capabilities: dict[str, compiler_enums.Capability] = {}\n\n for name, role_desc in self._roles.items():\n superuser = bool(role_desc.get('superuser'))\n available_permissions = (role_desc.get('all_permissions') or ())\n\n if superuser:\n capability = compiler_enums.Capability.ALL\n else:\n capability = (\n compiler_enums.Capability.TRANSACTION\n | compiler_enums.Capability.SESSION_CONFIG\n | compiler_enums.Capability.PERSISTENT_CONFIG\n )\n\n # Non-superuser can be given capabilities via\n # the permissions\n if 'sys::perm::data_modification' in available_permissions:\n capability |= compiler_enums.Capability.MODIFICATIONS\n if 'sys::perm::ddl' in available_permissions:\n capability |= compiler_enums.Capability.DDL\n if 'sys::perm::branch_config' in available_permissions:\n capability |= compiler_enums.Capability.BRANCH_CONFIG\n if 'sys::perm::sql_session_config' in available_permissions:\n capability |= compiler_enums.Capability.SQL_SESSION_CONFIG\n if 'sys::perm::analyze' in available_permissions:\n capability |= compiler_enums.Capability.ANALYZE\n\n role_capabilities[name] = capability\n\n self._role_capabilities = immutables.Map(role_capabilities)\n\n async def _fetch_roles(self, syscon: pgcon.PGConnection) -> None:\n role_query = self._server.get_sys_query(\"roles\")\n json_data = await syscon.sql_fetch_val(role_query, use_prep_stmt=True)\n roles = json.loads(json_data)\n self._roles = immutables.Map([(r[\"name\"], r) for r in roles])\n self._refresh_role_capabilities()\n\n async def init_sys_pgcon(self) -> None:\n self._sys_pgcon_waiter = asyncio.Lock()\n self.__sys_pgcon = await self._pg_connect(\n defines.EDGEDB_SYSTEM_DB,\n source_description=\"init_sys_pgcon\",\n )\n self._sys_pgcon_last_active_time = time.monotonic()\n self._sys_pgcon_ready_evt = asyncio.Event()\n self._sys_pgcon_reconnect_evt = asyncio.Event()\n\n async def get_patch_count(self, conn: pgcon.PGConnection) -> int:\n \"\"\"Get the number of applied patches.\"\"\"\n num_patches = await instdata.get_instdata(\n conn, 'num_patches', 'json')\n res: int = json.loads(num_patches) if num_patches else 0\n return res\n\n async def _check_metaschema_compatibility(\n self, con: pgcon.PGConnection\n ) -> None:\n from edb.pgsql import patches as pg_patches\n\n # Check catalog version\n result = await instdata.get_instdata(\n con, 'instancedata', 'json', versioned=False\n )\n catver = json.loads(result).get('catver')\n if catver != defines.EDGEDB_CATALOG_VERSION:\n raise errors.ConfigurationError(\n 'database instance incompatible with this version of Gel',\n details=(\n f'The database instance was initialized with '\n f'Gel format version {catver}, but this version '\n f'of the server expects format version '\n f'{defines.EDGEDB_CATALOG_VERSION}.'\n ),\n hint=(\n 'You need to either recreate the instance and upgrade '\n 'using dump/restore, or do an inplace upgrade.'\n )\n )\n\n # Check patch count\n num_patches = await self.get_patch_count(con)\n if num_patches < len(pg_patches.PATCHES):\n raise errors.ConfigurationError(\n 'database instance incompatible with this version of Gel',\n details=f\"expected {len(pg_patches.PATCHES)} patches, \"\n f\"but only {num_patches} applied\",\n hint=\"if you are adding an old backend to a multi-tenant \"\n \"server, firstly run a new single-tenant server on \"\n \"that backend to apply the patches.\",\n )\n\n async def init(self, compat_check: bool = False) -> None:\n logger.debug(\"starting database introspection\")\n async with self.use_sys_pgcon() as syscon:\n if compat_check:\n await self._check_metaschema_compatibility(syscon)\n result = await instdata.get_instdata(\n syscon, 'instancedata', 'json')\n self._instance_data = immutables.Map(json.loads(result))\n await self._fetch_roles(syscon)\n if self._server.get_compiler_pool() is None:\n # Parse global schema in I/O process if this is done only once\n logger.debug(\"parsing global schema locally\")\n global_schema_pickle = pickle.dumps(\n await self._server.introspect_global_schema(syscon), -1\n )\n data = None\n else:\n # Multi-tenant server defers the parsing into the compiler\n data = await self._server.introspect_global_schema_json(syscon)\n compiler_pool = self._server.get_compiler_pool()\n\n default_database = await instdata.get_instdata(\n syscon, 'default_branch', 'text')\n if default_database:\n self.default_database = default_database.decode('utf-8')\n\n if data is not None:\n logger.debug(\"parsing global schema\")\n global_schema_pickle = (\n await compiler_pool.parse_global_schema(data)\n )\n\n logger.info(\"loading system config\")\n sys_config = await self._load_sys_config()\n default_sysconfig = await self._load_sys_config(\"sysconfig_default\")\n await self._load_reported_config()\n\n # To make in-place upgrade failures more testable, check\n # 'force_database_error' with a 'startup' scope.\n force_error = self._server.config_lookup(\n 'force_database_error', sys_config)\n edbcompiler.maybe_force_database_error(force_error, scope='startup')\n\n self._dbindex = dbview.DatabaseIndex(\n self,\n std_schema=self._server.get_std_schema(),\n global_schema_pickle=global_schema_pickle,\n sys_config=sys_config,\n default_sysconfig=default_sysconfig,\n sys_config_spec=self._server.config_settings,\n )\n\n await self._introspect_dbs()\n\n await self.load_extension_packages(buildmeta.get_extension_dir_path())\n # Allow user-specified too.\n for dir in self._extensions_dirs:\n await self.load_extension_packages(dir)\n\n # Now, once all DBs have been introspected, start listening on\n # any notifications about schema/roles/etc changes.\n assert self.__sys_pgcon is not None\n await self.__sys_pgcon.listen_for_sysevent()\n self.__sys_pgcon.mark_as_system_db()\n self._sys_pgcon_ready_evt.set()\n\n self.populate_sys_auth()\n self.reload_readiness_state()\n self._initing = False\n\n async def load_extension_packages(self, path: pathlib.Path) -> None:\n exts = []\n if self._is_extension_package(path):\n exts.append(path)\n else:\n try:\n with os.scandir(path) as it:\n for entry in it:\n if (\n entry.is_dir()\n and self._is_extension_package(entry)\n ):\n exts.append(pathlib.Path(entry))\n except FileNotFoundError:\n pass\n\n if not exts:\n return\n\n async with self.use_sys_pgcon() as syscon:\n from edb.pgsql import trampoline\n ext_packages_json = await syscon.sql_fetch_val(\n trampoline.fixup_query(\"\"\"\n SELECT json_agg(o.c)\n FROM (\n SELECT\n json_build_array(p.name, p.version) AS c\n FROM\n edgedb_VER.\"_SysExtensionPackage\" AS p\n ) AS o;\n \"\"\").encode('utf-8')\n )\n ext_packages = {\n (name, verutils.from_json(version))\n for name, version in json.loads(ext_packages_json)\n }\n\n for ext in exts:\n await self._load_extension_package(ext, ext_packages)\n\n def _is_extension_package(self, path: pathlib.Path | os.DirEntry) -> bool:\n return (pathlib.Path(path) / 'MANIFEST.toml').exists()\n\n async def _load_extension_package(\n self,\n path: pathlib.Path,\n ext_packages: set[tuple[str, verutils.Version]],\n ) -> None:\n with open(path / 'MANIFEST.toml', 'rb') as m:\n manifest = tomllib.load(m)\n\n name = manifest['name']\n version = verutils.parse_version(manifest['version'])\n if (name, version) in ext_packages:\n logger.info(\n f\"Extension package '{manifest['name']}' {version} \"\n f\"already installed\"\n )\n\n return\n\n scripts = []\n for file in manifest['files']:\n with open(path / file, 'rb') as f:\n scripts.append(f.read().decode('utf-8'))\n\n from edb.schema import schema as s_schema\n\n async with self.use_sys_pgcon() as syscon:\n global_schema = await self._server.introspect_global_schema(syscon)\n compiler = edbcompiler.new_compiler(\n std_schema=self._server._std_schema,\n reflection_schema=self._server._refl_schema,\n schema_class_layout=self._server._schema_class_layout,\n )\n compilerctx = edbcompiler.new_compiler_context(\n compiler_state=compiler.state,\n global_schema=global_schema,\n user_schema=s_schema.EMPTY_SCHEMA,\n internal_schema_mode=True,\n # Extension installation only works if stdmode or testmode is\n # set. Force testmode to be set, since we don't want to set\n # stdmode, because we want any externally loaded extensions to\n # be marked as *not* builtin.\n force_testmode=True,\n )\n\n script = '\\n'.join(scripts)\n _, sql_script = edbcompiler.compile_edgeql_script(compilerctx, script)\n logger.info(\n f\"Installing extension package '{manifest['name']}'\")\n async with self.use_sys_pgcon() as syscon:\n await syscon.sql_execute(sql_script.encode('utf-8'))\n global_schema = await self._server.introspect_global_schema(syscon)\n\n assert self._dbindex\n self._dbindex.update_global_schema(pickle.dumps(global_schema))\n\n def start_watching_files(self):\n if self._readiness_state_file is not None:\n\n def reload_state_file():\n self.reload_readiness_state()\n\n self._file_watch_finalizers.append(\n self._server.monitor_fs(\n self._readiness_state_file, reload_state_file\n )\n )\n\n if self._jwt_sub_allowlist_file is not None:\n\n def reload_jwt_sub_allowlist_file():\n self.load_jwt_sub_allowlist()\n\n self._file_watch_finalizers.append(\n self._server.monitor_fs(\n self._jwt_sub_allowlist_file, reload_jwt_sub_allowlist_file\n )\n )\n\n if self._jwt_revocation_list_file is not None:\n\n def reload_jwt_revocation_list_file():\n self.load_jwt_revocation_list()\n\n self._file_watch_finalizers.append(\n self._server.monitor_fs(\n self._jwt_revocation_list_file,\n reload_jwt_revocation_list_file,\n )\n )\n\n if self._config_file is not None:\n\n def reload_config_file():\n self.reload_config_file.schedule()\n\n self._file_watch_finalizers.append(\n self.server.monitor_fs(self._config_file, reload_config_file)\n )\n\n async def start_accepting_new_tasks(self) -> None:\n assert self._task_group is None\n self._task_group = asyncio.TaskGroup()\n await self._task_group.__aenter__()\n self._accept_new_tasks = True\n await self._cluster.start_watching(self.on_switch_over)\n\n def start_running(self) -> None:\n self._running = True\n self._accepting_connections = True\n assert self._dbindex is not None\n for db in self._dbindex.iter_dbs():\n db.start_stop_extensions()\n\n def stop_accepting_connections(self) -> None:\n self._accepting_connections = False\n\n @property\n def accept_new_tasks(self):\n return self._accept_new_tasks\n\n def is_db_ready(self, dbname: str) -> bool:\n if not self._accept_new_tasks:\n return False\n\n if (\n not (db := self.maybe_get_db(dbname=dbname))\n or not db.is_introspected()\n ):\n return False\n\n return True\n\n def create_task(\n self,\n coro: Coroutine,\n *,\n interruptable: bool,\n name: Optional[str] = None,\n ) -> asyncio.Task:\n # Interruptable tasks are regular asyncio tasks that may be interrupted\n # randomly in the middle when the event loop stops; while tasks with\n # interruptable=False are always awaited before the server stops, so\n # that e.g. all finally blocks get a chance to execute in those tasks.\n # Therefore, it is an error trying to create a task while the server is\n # not expecting one, so always couple the call with an additional check\n if self._accept_new_tasks and self._task_group is not None:\n current_tenant.set(self.get_instance_name())\n if interruptable:\n rv = self.__loop.create_task(coro, name=name)\n else:\n rv = self._task_group.create_task(coro, name=name)\n\n # Keep a strong reference of the created Task\n if name is not None:\n if name in self._named_tasks:\n raise RuntimeError(\n f\"task {name!r} already exists on on this server\")\n self._named_tasks[name] = rv\n rv.add_done_callback(\n lambda task: self._named_tasks.pop(task.get_name(), None))\n else:\n self._tasks.add(rv)\n rv.add_done_callback(self._tasks.discard)\n\n return rv\n else:\n # Hint: add `if tenant.accept_new_tasks` before `.create_task()`\n raise RuntimeError(\"task cannot be created at this time\")\n\n def get_task(self, name: str) -> Optional[asyncio.Task]:\n return self._named_tasks.get(name)\n\n def stop(self) -> None:\n self._running = False\n self._accept_new_tasks = False\n self._cluster.stop_watching()\n self._stop_watching_files()\n self._server.request_frontend_stop(self)\n\n def _stop_watching_files(self):\n while self._file_watch_finalizers:\n self._file_watch_finalizers.pop()()\n\n async def wait_stopped(self) -> None:\n if self._task_group is not None:\n tg = self._task_group\n self._task_group = None\n await tg.__aexit__(*sys.exc_info())\n await self._pg_pool.close()\n\n def terminate_sys_pgcon(self) -> None:\n if self.__sys_pgcon is not None:\n self.__sys_pgcon.terminate()\n self.__sys_pgcon = None\n del self._sys_pgcon_waiter\n\n def set_init_con_data(self, data: list[config.ConState]) -> None:\n self._init_con_data = data\n self._init_con_sql = None\n if data:\n self._init_con_sql = self._make_init_con_sql(data)\n\n def _make_init_con_sql(self, data: list[config.ConState]) -> bytes:\n if not data:\n return b\"\"\n\n from edb.pgsql import common\n\n quoted_json = common.quote_literal(json.dumps(data))\n return textwrap.dedent(\n f'''\n INSERT INTO _edgecon_state\n SELECT * FROM jsonb_to_recordset({quoted_json}::jsonb)\n AS cfg(name text, value jsonb, type text);\n '''\n ).strip().encode()\n\n async def _pg_connect(\n self,\n dbname: str,\n source_description: str=\"pool connection\"\n ) -> pgcon.PGConnection:\n ha_serial = self._ha_master_serial\n if self.get_backend_runtime_params().has_create_database:\n pg_dbname = self.get_pg_dbname(dbname)\n else:\n pg_dbname = self.get_pg_dbname(defines.EDGEDB_SUPERUSER_DB)\n started_at = time.monotonic()\n try:\n rv = await self._cluster.connect(\n source_description=source_description,\n database=pg_dbname,\n apply_init_script=True\n )\n if self._server.stmt_cache_size is not None:\n rv.set_stmt_cache_size(self._server.stmt_cache_size)\n\n if self._init_con_sql:\n await rv.sql_execute(self._init_con_sql)\n rv.last_init_con_data = self._init_con_data\n\n except Exception:\n metrics.backend_connection_establishment_errors.inc(\n 1.0, self._instance_name\n )\n raise\n finally:\n metrics.backend_connection_establishment_latency.observe(\n time.monotonic() - started_at, self._instance_name\n )\n if ha_serial == self._ha_master_serial:\n rv.set_tenant(self)\n if self._backend_adaptive_ha is not None:\n self._backend_adaptive_ha.on_pgcon_made(\n dbname == defines.EDGEDB_SYSTEM_DB\n )\n metrics.total_backend_connections.inc(1.0, self._instance_name)\n metrics.current_backend_connections.inc(1.0, self._instance_name)\n return rv\n else:\n rv.terminate()\n raise ConnectionError(\"connected to outdated Postgres master\")\n\n async def _pg_disconnect(self, conn: pgcon.PGConnection) -> None:\n metrics.current_backend_connections.dec(1.0, self._instance_name)\n conn.terminate()\n\n def get_introspection_lock(\n self,\n dbname: str,\n ) -> asyncio.Lock:\n lock = self._introspection_locks.get(dbname)\n if not lock:\n self._introspection_locks[dbname] = lock = asyncio.Lock()\n return lock\n\n @contextlib.asynccontextmanager\n async def direct_pgcon(\n self,\n dbname: str,\n ) -> AsyncGenerator[pgcon.PGConnection, None]:\n conn = None\n try:\n conn = await self._pg_connect(\n dbname,\n source_description=\"direct_pgcon\"\n )\n yield conn\n finally:\n if conn is not None:\n await self._pg_disconnect(conn)\n\n @contextlib.asynccontextmanager\n async def use_sys_pgcon(self) -> AsyncGenerator[pgcon.PGConnection, None]:\n if not self._initing and not self._running:\n raise RuntimeError(\"Gel server is not running.\")\n\n await self._sys_pgcon_waiter.acquire()\n\n if not self._initing and not self._running:\n self._sys_pgcon_waiter.release()\n raise RuntimeError(\"Gel server is not running.\")\n\n if self.__sys_pgcon is None or not self.__sys_pgcon.is_healthy():\n conn, self.__sys_pgcon = self.__sys_pgcon, None\n if conn is not None:\n self._sys_pgcon_ready_evt.clear()\n conn.abort()\n # We depend on the reconnect on connection_lost() of __sys_pgcon\n await self._sys_pgcon_ready_evt.wait()\n if self.__sys_pgcon is None:\n self._sys_pgcon_waiter.release()\n raise RuntimeError(\"Cannot acquire pgcon to the system DB.\")\n\n try:\n yield self.__sys_pgcon\n finally:\n if self.__sys_pgcon is not None and self.__sys_pgcon.is_healthy():\n self._sys_pgcon_last_active_time = time.monotonic()\n self._sys_pgcon_waiter.release()\n\n def set_stmt_cache_size(self, size: int) -> None:\n for conn in self._pg_pool.iterate_connections():\n conn.set_stmt_cache_size(size)\n\n def on_sys_pgcon_parameter_status_updated(\n self,\n name: str,\n value: str,\n ) -> None:\n try:\n if name == \"in_hot_standby\" and value == \"on\":\n # It is a strong evidence of failover if the sys_pgcon receives\n # a notification that in_hot_standby is turned on.\n self.on_sys_pgcon_failover_signal()\n except Exception:\n metrics.background_errors.inc(\n 1.0,\n self._instance_name,\n \"on_sys_pgcon_parameter_status_updated\"\n )\n raise\n\n def on_sys_pgcon_failover_signal(self) -> None:\n if not self._running:\n return\n try:\n if self._backend_adaptive_ha is not None:\n # Switch to FAILOVER if adaptive HA is enabled\n self._backend_adaptive_ha.set_state_failover()\n elif getattr(self._cluster, \"_ha_backend\", None) is None:\n # If the server is not using an HA backend, nor has enabled the\n # adaptive HA monitoring, we still try to \"switch over\" by\n # disconnecting all pgcons if failover signal is received,\n # allowing reconnection to happen sooner.\n self.on_switch_over()\n # Else, the HA backend should take care of calling on_switch_over()\n except Exception:\n metrics.background_errors.inc(\n 1.0, self._instance_name, \"on_sys_pgcon_failover_signal\"\n )\n raise\n\n def on_sys_pgcon_connection_lost(self, exc: Exception | None) -> None:\n try:\n if not self._running:\n # The tenant is shutting down, release all events so that\n # the waiters if any could continue and exit\n self._sys_pgcon_ready_evt.set()\n self._sys_pgcon_reconnect_evt.set()\n return\n\n logger.error(\n \"Connection to the system database is \"\n + (\"closed.\" if exc is None else f\"broken! Reason: {exc}\")\n )\n self.set_pg_unavailable_msg(\n \"Connection is lost, please check server log for the reason.\"\n )\n self.__sys_pgcon = None\n self._sys_pgcon_ready_evt.clear()\n if self._accept_new_tasks:\n self.create_task(\n self._reconnect_sys_pgcon(), interruptable=True\n )\n self.on_pgcon_broken(True)\n except Exception:\n metrics.background_errors.inc(\n 1.0, self._instance_name, \"on_sys_pgcon_connection_lost\"\n )\n raise\n\n async def _reconnect_sys_pgcon(self) -> None:\n try:\n conn = None\n while self._running:\n # Keep retrying as far as:\n # 1. This tenant is still running\n # 2. We still cannot connect to the Postgres cluster\n try:\n conn = await self._pg_connect(\n defines.EDGEDB_SYSTEM_DB,\n source_description=\"_reconnect_sys_pgcon\"\n )\n break\n except OSError:\n pass\n except pgcon_errors.BackendError as e:\n # Be quiet if the Postgres cluster is still starting up,\n # or the HA failover is still in progress.\n # TODO: ERROR_FEATURE_NOT_SUPPORTED should be removed\n # once PostgreSQL supports SERIALIZABLE in hot standbys\n if not (\n e.code_is(pgcon_errors.ERROR_FEATURE_NOT_SUPPORTED)\n or e.code_is(pgcon_errors.ERROR_CANNOT_CONNECT_NOW)\n or e.code_is(\n pgcon_errors.ERROR_READ_ONLY_SQL_TRANSACTION\n )\n ):\n logger.error(\"Failed connecting to the backend: %s\", e)\n\n if self._running:\n logger.info(\"Waiting for the backend to recover\")\n try:\n # Retry after INTERVAL seconds, unless the event is set\n # and we can retry immediately after the event.\n await asyncio.wait_for(\n self._sys_pgcon_reconnect_evt.wait(),\n defines.SYSTEM_DB_RECONNECT_INTERVAL,\n )\n # But the event can only skip one INTERVAL.\n self._sys_pgcon_reconnect_evt.clear()\n except asyncio.TimeoutError:\n pass\n\n if not self._running:\n if conn is not None:\n conn.abort()\n return\n\n assert conn is not None\n logger.info(\"Successfully reconnected to the system database.\")\n self.__sys_pgcon = conn\n self.__sys_pgcon.mark_as_system_db()\n self._sys_pgcon_last_active_time = time.monotonic()\n # This await is meant to be after mark_as_system_db() because we\n # need the pgcon to be able to trigger another reconnect if its\n # connection is lost during this await.\n await self.__sys_pgcon.listen_for_sysevent()\n self.set_pg_unavailable_msg(None)\n finally:\n self._sys_pgcon_ready_evt.set()\n\n def on_pgcon_broken(self, is_sys_pgcon: bool = False) -> None:\n try:\n if self._backend_adaptive_ha:\n self._backend_adaptive_ha.on_pgcon_broken(is_sys_pgcon)\n except Exception:\n metrics.background_errors.inc(\n 1.0, self._instance_name, \"on_pgcon_broken\"\n )\n raise\n\n def on_pgcon_lost(self) -> None:\n try:\n if self._backend_adaptive_ha:\n self._backend_adaptive_ha.on_pgcon_lost()\n except Exception:\n metrics.background_errors.inc(\n 1.0, self._instance_name, \"on_pgcon_lost\")\n raise\n\n def set_pg_unavailable_msg(self, msg: str | None) -> None:\n if msg is None or self._pg_unavailable_msg is None:\n self._pg_unavailable_msg = msg\n\n @contextlib.asynccontextmanager\n async def with_pgcon(\n self, dbname: str, *,\n discard: bool=False\n ) -> AsyncGenerator[pgcon.PGConnection, None]:\n conn = await self.acquire_pgcon(dbname=dbname)\n try:\n yield conn\n finally:\n self.release_pgcon(dbname, conn, discard=discard)\n\n async def acquire_pgcon(self, dbname: str) -> pgcon.PGConnection:\n if self._pg_unavailable_msg is not None:\n raise errors.BackendUnavailableError(\n \"Postgres is not available: \" + self._pg_unavailable_msg\n )\n\n for _ in range(self._pg_pool.max_capacity):\n conn = await self._pg_pool.acquire(dbname)\n if not conn.is_healthy():\n logger.warning(\"acquired an unhealthy pgcon; discard now\")\n elif conn.last_init_con_data is not self._init_con_data:\n try:\n await conn.sql_execute(\n pgcon.RESET_STATIC_CFG_SCRIPT +\n (self._init_con_sql or b'')\n )\n except Exception as e:\n logger.warning(\n \"failed to update pgcon; discard now: %s\", e\n )\n else:\n conn.last_init_con_data = self._init_con_data\n return conn\n else:\n return conn\n self._pg_pool.release(dbname, conn, discard=True)\n else:\n # This is unlikely to happen, but we defer to the caller to retry\n # when it does happen\n raise errors.BackendUnavailableError(\n \"No healthy backend connection available at the moment, \"\n \"please try again.\"\n )\n\n def release_pgcon(\n self,\n dbname: str,\n conn: pgcon.PGConnection,\n *,\n discard: bool = False,\n ) -> None:\n if not conn.is_healthy():\n if not discard:\n logger.warning(\"Released an unhealthy pgcon; discard now.\")\n discard = True\n try:\n self._pg_pool.release(dbname, conn, discard=discard)\n except Exception:\n metrics.background_errors.inc(\n 1.0, self._instance_name, \"release_pgcon\"\n )\n raise\n\n def allow_database_connections(self, dbname: str) -> None:\n self._block_new_connections.discard(dbname)\n\n def is_database_connectable(self, dbname: str) -> bool:\n return (\n self._running\n and dbname != defines.EDGEDB_TEMPLATE_DB\n and dbname not in self._block_new_connections\n )\n\n async def ensure_database_not_connected(\n self, dbname: str, close_frontend_conns: bool = False\n ) -> None:\n if self._dbindex and self._dbindex.count_connections(dbname):\n if close_frontend_conns:\n self._server.request_stop_fe_conns(dbname)\n else:\n # If there are open Gel connections to the `dbname` DB\n # just raise the error Postgres would have raised itself.\n raise errors.ExecutionError(\n f\"database branch {dbname!r} is being accessed by \"\n f\"other users\"\n )\n\n self._block_new_connections.add(dbname)\n\n rloop = retryloop.RetryLoop(\n timeout=30.0,\n ignore=errors.ExecutionError,\n )\n\n async for iteration in rloop:\n async with iteration:\n # Signal adjacent servers to prune their connections to this\n # database.\n await self.signal_sysevent(\n \"ensure-database-not-used\", dbname=dbname\n )\n\n # Prune our inactive connections. (Do it in the loop\n # to help in the close_frontend_conns situation.)\n await self._pg_pool.prune_inactive_connections(dbname)\n\n await self._pg_ensure_database_not_connected(dbname)\n\n async def _pg_ensure_database_not_connected(self, dbname: str) -> None:\n async with self.use_sys_pgcon() as pgcon:\n conns = await pgcon.sql_fetch_col(\n b\"\"\"\n SELECT\n row_to_json(pg_stat_activity)\n FROM\n pg_stat_activity\n WHERE\n datname = $1\n \"\"\",\n args=[self.get_pg_dbname(dbname).encode(\"utf-8\")],\n )\n\n if conns:\n debug_info = \"\"\n if self.server.in_dev_mode() or self.server.in_test_mode():\n jconns = [json.loads(conn) for conn in conns]\n debug_info = \": \" + json.dumps(jconns)\n\n raise errors.ExecutionError(\n f\"database branch {dbname!r} is being accessed by \"\n f\"other users{debug_info}\"\n )\n\n @contextlib.asynccontextmanager\n async def _with_intro_pgcon(\n self, dbname: str\n ) -> AsyncGenerator[pgcon.PGConnection | None, None]:\n conn = None\n try:\n conn = await self.acquire_pgcon(dbname)\n yield conn\n except pgcon_errors.BackendError as e:\n if e.code_is(pgcon_errors.ERROR_INVALID_CATALOG_NAME):\n # database does not exist (anymore)\n logger.warning(\n \"Detected concurrently-dropped database branch %s; \"\n \"skipping.\",\n dbname,\n )\n if self._dbindex is not None and self._dbindex.has_db(dbname):\n self._dbindex.unregister_db(dbname)\n yield None\n else:\n raise\n finally:\n if conn:\n self.release_pgcon(dbname, conn)\n\n async def _introspect_extensions(\n self, conn: pgcon.PGConnection\n ) -> set[str]:\n from edb.pgsql import trampoline\n extension_names_json = await conn.sql_fetch_val(\n trampoline.fixup_query(\"\"\"\n SELECT json_agg(name) FROM edgedb_VER.\"_SchemaExtension\";\n \"\"\").encode('utf-8'),\n )\n if extension_names_json:\n extensions = set(json.loads(extension_names_json))\n else:\n extensions = set()\n\n return extensions\n\n async def _debug_introspect(\n self,\n conn: pgcon.PGConnection,\n global_schema_pickle,\n ) -> Any:\n user_schema_json = (\n await self._server.introspect_user_schema_json(conn)\n )\n db_config_json = await self._server.introspect_db_config(conn)\n\n compiler_pool = self._server.get_compiler_pool()\n return (await compiler_pool.parse_user_schema_db_config(\n user_schema_json, db_config_json, global_schema_pickle,\n )).user_schema_pickle\n\n async def introspect_db(\n self,\n dbname: str,\n *,\n conn: Optional[pgcon.PGConnection]=None,\n reintrospection: bool=False,\n ) -> None:\n \"\"\"Use this method to (re-)introspect a DB.\n\n If the DB is already registered in self._dbindex, its\n schema, config, etc. would simply be updated. If it's missing\n an entry for it would be created.\n\n All remote notifications of remote events should use this method\n to refresh the state. Even if the remote event was a simple config\n change, a lot of other events could happen before it was sent to us\n by a remote server and us receiving it. E.g. a DB could have been\n dropped and recreated again. It's safer to refresh the entire state\n than refreshing individual components of it. Besides, DDL and\n database-level config modifications are supposed to be rare events.\n\n This supports passing in a connection to use as well, so that\n we can synchronously introspect on config changes without\n risking deadlock by acquiring two connections at once.\n\n Returns True if the query cache mode changed.\n\n \"\"\"\n cm = (\n contextlib.nullcontext(conn) if conn\n else self._with_intro_pgcon(dbname)\n )\n\n async with cm as conn:\n if not conn:\n return\n # Acquire a per-db lock for doing the introspection, to avoid\n # race conditions where an older introspection might overwrite\n # a newer one.\n async with self.get_introspection_lock(dbname):\n await self._introspect_db(\n dbname, conn=conn, reintrospection=reintrospection\n )\n\n async def _introspect_db(\n self,\n dbname: str,\n conn: pgcon.PGConnection,\n reintrospection: bool,\n ) -> None:\n from edb.pgsql import trampoline\n logger.info(\"introspecting database '%s'\", dbname)\n\n assert self._dbindex is not None\n if db := self._dbindex.maybe_get_db(dbname):\n cache_mode_val = db.lookup_config('query_cache_mode')\n else:\n cache_mode_val = self._dbindex.lookup_config('query_cache_mode')\n old_cache_mode = config.QueryCacheMode.effective(cache_mode_val)\n\n # Introspection\n user_schema_json = (\n await self._server.introspect_user_schema_json(conn)\n )\n\n reflection_cache_json = await conn.sql_fetch_val(\n trampoline.fixup_query(\"\"\"\n SELECT json_agg(o.c)\n FROM (\n SELECT\n json_build_object(\n 'eql_hash', t.eql_hash,\n 'argnames', array_to_json(t.argnames)\n ) AS c\n FROM\n ROWS FROM(edgedb_VER._get_cached_reflection())\n AS t(eql_hash text, argnames text[])\n ) AS o;\n \"\"\").encode('utf-8'),\n )\n\n reflection_cache = immutables.Map(\n {\n r[\"eql_hash\"]: tuple(r[\"argnames\"])\n for r in json.loads(reflection_cache_json)\n }\n )\n\n backend_ids_json = await conn.sql_fetch_val(\n trampoline.fixup_query(\"\"\"\n SELECT\n json_object_agg(\n \"id\"::text,\n json_build_array(\"backend_id\", \"name\")\n )::text\n FROM\n edgedb_VER.\"_SchemaType\"\n \"\"\").encode('utf-8'),\n )\n backend_ids = json.loads(backend_ids_json)\n\n db_config_json = await self._server.introspect_db_config(conn)\n\n extensions = await self._introspect_extensions(conn)\n\n query_cache: list[tuple[bytes, ...]] | None = None\n if (\n not reintrospection\n and old_cache_mode is not config.QueryCacheMode.InMemory\n ):\n query_cache = await self._load_query_cache(conn)\n\n # Analysis\n compiler_pool = self._server.get_compiler_pool()\n parsed_db = await compiler_pool.parse_user_schema_db_config(\n user_schema_json, db_config_json, self.get_global_schema_pickle()\n )\n db = self._dbindex.register_db(\n dbname,\n user_schema_pickle=parsed_db.user_schema_pickle,\n schema_version=parsed_db.schema_version,\n db_config=parsed_db.database_config,\n reflection_cache=reflection_cache,\n backend_ids=backend_ids,\n extensions=extensions,\n ext_config_settings=parsed_db.ext_config_settings,\n feature_used_metrics=parsed_db.feature_used_metrics,\n )\n db.set_state_serializer(\n parsed_db.protocol_version,\n parsed_db.state_serializer,\n )\n cache_mode = config.QueryCacheMode.effective(\n db.lookup_config('query_cache_mode')\n )\n\n if query_cache and cache_mode is not config.QueryCacheMode.InMemory:\n db.hydrate_cache(query_cache)\n elif old_cache_mode is not cache_mode:\n logger.info(\n \"clearing query cache for database '%s'\", dbname)\n await conn.sql_execute(\n b'SELECT edgedb._clear_query_cache()')\n assert self._dbindex\n self._dbindex.get_db(dbname).clear_query_cache()\n\n async def _early_introspect_db(self, dbname: str) -> None:\n \"\"\"We need to always introspect the extensions for each database.\n\n Otherwise, we won't know to accept connections for graphql or\n http, for example, until a native connection is made.\n \"\"\"\n current_tenant.set(self.get_instance_name())\n logger.info(\"introspecting extensions for database '%s'\", dbname)\n\n async with self._with_intro_pgcon(dbname) as conn:\n if not conn:\n return\n assert self._dbindex is not None\n if not self._dbindex.has_db(dbname):\n extensions = await self._introspect_extensions(conn)\n # Re-check in case we have a concurrent introspection task.\n if not self._dbindex.has_db(dbname):\n self._dbindex.register_db(\n dbname,\n user_schema_pickle=None,\n schema_version=None,\n db_config=None,\n reflection_cache=None,\n backend_ids=None,\n extensions=extensions,\n ext_config_settings=None,\n early=True,\n )\n\n # Early introspection runs *before* we start accepting tasks.\n # This means that if we are one of multiple frontends, and we\n # get a ensure-database-not-used message, we aren't able to\n # handle it. This can result in us hanging onto a connection\n # that another frontend wants to get rid of.\n #\n # We still want to use the pool, though, since it limits our\n # connections in the way we want.\n #\n # Hack around this by pruning the connection ourself.\n await self._pg_pool.prune_inactive_connections(dbname)\n\n async def _introspect_dbs(self) -> None:\n async with self.use_sys_pgcon() as syscon:\n dbnames = await self._server.get_dbnames(syscon)\n\n async with asyncio.TaskGroup() as g:\n for dbname in dbnames:\n # There's a risk of the DB being dropped by another server\n # between us building the list of databases and loading\n # information about them.\n g.create_task(self._early_introspect_db(dbname))\n\n async def _load_reported_config(self) -> None:\n async with self.use_sys_pgcon() as syscon:\n try:\n data = await syscon.sql_fetch_val(\n self._server.get_sys_query(\"report_configs\"),\n use_prep_stmt=True,\n state=b'[]', # clear _config_cache\n )\n\n for (\n protocol_ver,\n typedesc,\n ) in self._server.get_report_config_typedesc().items():\n self._report_config_data[protocol_ver] = (\n struct.pack(\"!L\", len(typedesc))\n + typedesc\n + struct.pack(\"!L\", len(data))\n + data\n )\n except Exception:\n metrics.background_errors.inc(\n 1.0, self._instance_name, \"load_reported_config\"\n )\n raise\n\n async def _load_sys_config(\n self,\n query_name: str = \"sysconfig\",\n syscon: pgcon.PGConnection | None = None,\n ) -> Mapping[str, config.SettingValue]:\n query = self._server.get_sys_query(query_name)\n if syscon is None:\n async with self.use_sys_pgcon() as syscon:\n sys_config_json = await syscon.sql_fetch_val(query)\n else:\n sys_config_json = await syscon.sql_fetch_val(query)\n\n return config.from_json(self._server.config_settings, sys_config_json)\n\n async def _reintrospect_global_schema(self) -> None:\n if not self._initing and not self._running:\n logger.warning(\n \"global-schema-changes event received during shutdown; \"\n \"ignoring.\"\n )\n return\n async with self.use_sys_pgcon() as syscon:\n data = await self._server.introspect_global_schema_json(syscon)\n await self._fetch_roles(syscon)\n compiler_pool = self._server.get_compiler_pool()\n global_schema_pickle = await compiler_pool.parse_global_schema(data)\n assert self._dbindex is not None\n self._dbindex.update_global_schema(global_schema_pickle)\n\n def populate_sys_auth(self) -> None:\n assert self._dbindex is not None\n cfg = self._dbindex.get_sys_config()\n auth = self._server.config_lookup(\"auth\", cfg) or ()\n self._sys_auth = tuple(sorted(auth, key=lambda a: a.priority))\n\n def resolve_branch_name(\n self, database: str | None, branch: str | None\n ) -> str:\n default = self.default_database\n if branch == '__default__':\n return default\n elif branch is not None:\n return branch\n elif (\n database == defines.EDGEDB_OLD_DEFAULT_DB\n and self.maybe_get_db(dbname=defines.EDGEDB_OLD_DEFAULT_DB) is None\n ):\n return default\n else:\n assert database is not None\n return database\n\n def resolve_user_name(self, user: str) -> str:\n if (\n user == defines.EDGEDB_OLD_SUPERUSER\n and user not in self.get_roles()\n ):\n return defines.EDGEDB_SUPERUSER\n else:\n return user\n\n async def get_auth_methods(\n self,\n user: str,\n transport: srvargs.ServerConnTransport,\n ) -> list[config.CompositeConfigType]:\n authlist = self._sys_auth\n methods = []\n\n if authlist:\n for auth in authlist:\n match = (user in auth.user or \"*\" in auth.user) and (\n not auth.method.transports\n or transport in auth.method.transports\n )\n\n if match:\n methods.append(auth.method)\n break\n\n if not methods:\n methods = self._server.get_default_auth_methods(transport)\n\n return methods\n\n async def new_dbview(\n self,\n *,\n dbname: str,\n query_cache: bool,\n protocol_version: defines.ProtocolVersion,\n role_name: str,\n ) -> dbview.DatabaseConnectionView:\n db = self.get_db(dbname=dbname)\n await db.introspection()\n assert self._dbindex is not None\n return self._dbindex.new_view(\n dbname,\n query_cache=query_cache,\n protocol_version=protocol_version,\n role_name=role_name,\n )\n\n def remove_dbview(self, dbview_: dbview.DatabaseConnectionView) -> None:\n assert self._dbindex is not None\n return self._dbindex.remove_view(dbview_)\n\n def schedule_reported_config_if_needed(self, setting_name: str) -> None:\n setting = self._server.config_settings.get(setting_name)\n if setting and setting.report and self._accept_new_tasks:\n self.create_task(self._load_reported_config(), interruptable=True)\n\n def load_jwcrypto(self, jwk_key: auth.JWKSet) -> None:\n self._jws_key = jwk_key\n self.load_jwt_sub_allowlist()\n self.load_jwt_revocation_list()\n\n def load_jwt_sub_allowlist(self) -> None:\n if self._jwt_sub_allowlist_file is not None:\n logger.info(\n \"(re-)loading JWT subject allowlist from \"\n f'\"{self._jwt_sub_allowlist_file}\"'\n )\n try:\n self._jwt_sub_allowlist = frozenset(\n self._jwt_sub_allowlist_file.read_text().splitlines(),\n )\n if self._jws_key is not None:\n self._jws_key.default_validation_context.allow(\n \"sub\", self._jwt_sub_allowlist\n )\n else:\n from . import server as edbserver\n raise edbserver.StartupError(\n \"cannot load JWT sub allowlist: no secret key\"\n )\n except Exception as e:\n from . import server as edbserver\n\n raise edbserver.StartupError(\n f\"cannot load JWT sub allowlist: {e}\"\n ) from e\n\n def load_jwt_revocation_list(self) -> None:\n if self._jwt_revocation_list_file is not None:\n logger.info(\n \"(re-)loading JWT revocation list from \"\n f'\"{self._jwt_revocation_list_file}\"'\n )\n try:\n self._jwt_revocation_list = frozenset(\n self._jwt_revocation_list_file.read_text().splitlines(),\n )\n if self._jws_key is not None:\n self._jws_key.default_validation_context.deny(\n \"jti\", self._jwt_revocation_list\n )\n else:\n from . import server as edbserver\n raise edbserver.StartupError(\n \"cannot load JWT revocation list: no secret key\"\n )\n except Exception as e:\n from . import server as edbserver\n\n raise edbserver.StartupError(\n f\"cannot load JWT revocation list: {e}\"\n ) from e\n\n def reload_readiness_state(self) -> None:\n if self._readiness_state_file is None:\n return\n try:\n with self._readiness_state_file.open(\"rt\") as rt:\n line = rt.readline().strip()\n try:\n state, _, reason = line.partition(\":\")\n self._readiness = srvargs.ReadinessState(state)\n self._readiness_reason = reason\n logger.info(\n \"readiness state file changed, \"\n \"setting server readiness to %r%s\",\n state,\n f\" ({reason})\" if reason else \"\",\n )\n except ValueError:\n logger.warning(\n \"invalid state in readiness state file (%r): %r, \"\n \"resetting server readiness to 'default'\",\n self._readiness_state_file,\n state,\n )\n self._readiness = srvargs.ReadinessState.Default\n\n except FileNotFoundError:\n logger.info(\n \"readiness state file (%s) removed, resetting \"\n \"server readiness to 'default'\",\n self._readiness_state_file,\n )\n self._readiness = srvargs.ReadinessState.Default\n\n except Exception as e:\n logger.warning(\n \"cannot read readiness state file (%s): %s, \"\n \"resetting server readiness to 'default'\",\n self._readiness_state_file,\n e,\n )\n self._readiness = srvargs.ReadinessState.Default\n\n self._accepting_connections = self.is_online()\n\n def set_readiness_state(self, state: srvargs.ReadinessState, reason: str):\n self._readiness = state\n self._readiness_reason = reason\n\n @asyncutil.exclusive_task\n async def reload_config_file(self):\n if self._config_file is None:\n return\n\n try:\n await self._reload_config_file()\n except Exception:\n logger.error(\"failed to reload config file\", exc_info=True)\n metrics.background_errors.inc(\n 1.0, self._instance_name, \"reload_config_file\"\n )\n\n async def load_config_file(self, compiler):\n logger.info(\"loading config file\")\n\n # Read the TOML file\n with self._config_file.open('rb') as f:\n toml_data = tomllib.load(f)\n\n # Handle special case for `magic_smtp_config`\n magic_smtp_config = toml_data.pop(\"magic_smtp_config\", None)\n if magic_smtp_config:\n await self.load_sidechannel_configs(\n magic_smtp_config, compiler=compiler\n )\n\n # Parse TOML config file content into JSON\n if toml_data and toml_data.get(\"cfg::Config\"):\n result = compiler.compile_structured_config(\n toml_data, \"configuration file\"\n )\n if asyncio.iscoroutine(result):\n result = await result\n\n def setting_filter(value: config.SettingValue) -> bool:\n if self._server.config_settings[value.name].backend_setting:\n raise errors.ConfigurationError(\n f\"backend config {value.name!r} cannot be set \"\n f\"via config file\"\n )\n return True\n\n json_obj = config.to_json_obj(\n self._server.config_settings,\n result[\"cfg::Config\"],\n include_source=False,\n setting_filter=setting_filter,\n )\n config_file_data = [\n {\n \"name\": name,\n \"value\": value,\n \"type\": config.ConStateType.config_file,\n }\n for name, value in json_obj.items()\n ]\n else:\n config_file_data = []\n\n # Update init_con_data and SQL\n self.set_init_con_data(\n [\n cs\n for cs in self._init_con_data\n if cs[\"type\"] != config.ConStateType.config_file\n ]\n + config_file_data\n )\n\n async def _reload_config_file(self):\n # Load TOML config file\n compiler = self._server.get_compiler_pool()\n await self.load_config_file(compiler)\n\n # Update sys pgcon and reload system config\n async with self.use_sys_pgcon() as syscon:\n if syscon.last_init_con_data is not self._init_con_data:\n await syscon.sql_execute(\n pgcon.RESET_STATIC_CFG_SCRIPT + (self._init_con_sql or b'')\n )\n syscon.last_init_con_data = self._init_con_data\n sys_config = await self._load_sys_config(syscon=syscon)\n # GOTCHA: no need to notify other EdgeDBs on the same backend about\n # such change to sysconfig, because static config is instance-local\n self._dbindex.update_sys_config(sys_config)\n\n def reload(self):\n # In multi-tenant mode, the file paths for the following states may be\n # unset in a reload, while it's impossible in a regular server.\n # Therefore, we are clearing the states here first, rather than doing\n # so in reload_readiness_state() or load_jwcrypto().\n self._readiness = srvargs.ReadinessState.Default\n self._jwt_sub_allowlist = None\n self._jwt_revocation_list = None\n\n # Re-add the fs watchers in case the path changed\n self._stop_watching_files()\n\n self.reload_readiness_state()\n self.load_jwcrypto()\n self.reload_config_file.schedule()\n\n self.start_watching_files()\n\n async def on_before_drop_db(\n self,\n dbname: str,\n current_dbname: str,\n close_frontend_conns: bool = False,\n ) -> None:\n if current_dbname == dbname:\n raise errors.ExecutionError(\n f\"cannot drop the currently open database branch {dbname!r}\"\n )\n\n await self.ensure_database_not_connected(\n dbname, close_frontend_conns=close_frontend_conns\n )\n\n async def on_before_create_db_from_template(\n self, dbname: str, current_dbname: str, mode: str\n ) -> None:\n # Make sure the database exists.\n # TODO: Is it worth producing a nicer error message if it\n # fails on the backside? (Because of a race?)\n self.get_db(dbname=dbname)\n\n if mode == 'TEMPLATE':\n await self.ensure_database_not_connected(dbname)\n\n async def on_after_create_db_from_template(\n self, tgt_dbname: str, src_dbname: str, mode: str\n ) -> None:\n if mode == 'TEMPLATE':\n self.allow_database_connections(tgt_dbname)\n return\n\n logger.info('Starting copy from %s to %s', src_dbname, tgt_dbname)\n from edb.pgsql import common\n from . import bootstrap # noqa: F402\n\n real_tgt_dbname = common.get_database_backend_name(\n tgt_dbname, tenant_id=self._tenant_id)\n real_src_dbname = common.get_database_backend_name(\n src_dbname, tenant_id=self._tenant_id)\n\n # HACK: Limit the maximum number of in-flight branch\n # creations. This is because branches use up to 3 concurrent\n # connections (one direct, two via pg_dump/pg_restore), and so\n # it can substantially blow our budget if many are in flight.\n # The right way to handle this issue would probably be to use\n # the connection pool to reserve the connections, but we would\n # need to carefully consider deadlock concerns if we want to\n # allow tasks to acquire multiple pool connections.\n async with self._branch_sem:\n async with self.direct_pgcon(tgt_dbname) as con:\n await bootstrap.create_branch(\n self._cluster,\n self._server._refl_schema,\n con,\n real_src_dbname,\n real_tgt_dbname,\n mode,\n self._server._sys_queries['backend_id_fixup'],\n )\n\n logger.info('Finished copy from %s to %s', src_dbname, tgt_dbname)\n\n def on_after_drop_db(self, dbname: str) -> None:\n try:\n assert self._dbindex is not None\n if self._dbindex.has_db(dbname):\n self._dbindex.unregister_db(dbname)\n self._block_new_connections.discard(dbname)\n except Exception:\n metrics.background_errors.inc(\n 1.0, self._instance_name, \"on_after_drop_db\"\n )\n raise\n\n async def ping_backend(self) -> bool:\n if not self._running:\n return False\n elapsed = time.monotonic() - self._sys_pgcon_last_active_time\n if elapsed > HEALTH_CHECK_MIN_INTERVAL:\n async with asyncio.timeout(HEALTH_CHECK_TIMEOUT):\n async with self.use_sys_pgcon() as syscon:\n await syscon.sql_fetch_val(b\"select 'OK'\")\n return True\n\n async def cancel_pgcon_operation(self, con: pgcon.PGConnection) -> bool:\n async with self.use_sys_pgcon() as syscon:\n if con.idle:\n # con could have received the query results while we\n # were acquiring a system connection to cancel it.\n return False\n\n if con.is_cancelling():\n # Somehow the connection is already being cancelled and\n # we don't want to have to cancellations go in parallel.\n return False\n\n con.start_pg_cancellation()\n try:\n # Returns True if the `pid` exists and it was able to send it a\n # SIGINT. Will throw an exception if the privileges aren't\n # sufficient.\n result = await syscon.sql_fetch_val(\n f\"SELECT pg_cancel_backend({con.backend_pid});\".encode(),\n )\n finally:\n con.finish_pg_cancellation()\n\n return result == b\"\\x01\"\n\n async def cancel_and_discard_pgcon(\n self,\n con: pgcon.PGConnection,\n dbname: str,\n ) -> None:\n try:\n if self._running:\n await self.cancel_pgcon_operation(con)\n finally:\n self.release_pgcon(dbname, con, discard=True)\n\n async def signal_sysevent(self, event: str, **kwargs) -> None:\n try:\n if not self._initing and not self._running:\n # This is very likely if we are doing\n # \"run_startup_script_and_exit()\", but is also possible if the\n # tenant was shut down with this coroutine as a background task\n # in flight.\n return\n\n async with self.use_sys_pgcon() as con:\n await con.signal_sysevent(event, **kwargs)\n except Exception:\n metrics.background_errors.inc(\n 1.0, self._instance_name, \"signal_sysevent\"\n )\n raise\n\n def on_remote_database_quarantine(self, dbname: str) -> None:\n if not self._accept_new_tasks:\n return\n\n # Block new connections to the database.\n self._block_new_connections.add(dbname)\n\n async def task():\n try:\n await self._pg_pool.prune_inactive_connections(dbname)\n except Exception:\n metrics.background_errors.inc(\n 1.0, self._instance_name, \"remote_db_quarantine\"\n )\n raise\n\n self.create_task(task(), interruptable=True)\n\n def on_remote_ddl(self, dbname: str) -> None:\n if not self.is_db_ready(dbname):\n return\n\n # Triggered by a postgres notification event 'schema-changes'\n # on the __edgedb_sysevent__ channel\n async def task():\n try:\n await self.introspect_db(dbname)\n except Exception:\n metrics.background_errors.inc(\n 1.0, self._instance_name, \"on_remote_ddl\"\n )\n raise\n\n self.create_task(task(), interruptable=True)\n\n def on_remote_database_changes(self) -> None:\n if not self._accept_new_tasks:\n return\n\n # Triggered by a postgres notification event 'database-changes'\n # on the __edgedb_sysevent__ channel\n async def task():\n async with self.use_sys_pgcon() as syscon:\n dbnames = set(await self._server.get_dbnames(syscon))\n\n tg = asyncio.TaskGroup()\n async with tg as g:\n for dbname in dbnames:\n if not self._dbindex.has_db(dbname):\n g.create_task(self._early_introspect_db(dbname))\n\n dropped = []\n for db in self._dbindex.iter_dbs():\n if db.name not in dbnames:\n dropped.append(db.name)\n for dbname in dropped:\n self.on_after_drop_db(dbname)\n\n self.create_task(task(), interruptable=True)\n\n def on_remote_database_config_change(self, dbname: str) -> None:\n if not self._accept_new_tasks:\n return\n\n # Triggered by a postgres notification event 'database-config-changes'\n # on the __edgedb_sysevent__ channel\n async def task():\n try:\n await self.introspect_db(dbname, reintrospection=True)\n except Exception:\n metrics.background_errors.inc(\n 1.0,\n self._instance_name,\n \"on_remote_database_config_change\",\n )\n raise\n\n self.create_task(task(), interruptable=True)\n\n async def process_local_database_config_change(\n self,\n conn: pgcon.PGConnection,\n dbname: str,\n ) -> None:\n # It's easier and safer to just do full re-introspection\n # of the DB and update all components of it.\n # TODO: Can we just do config?\n await self.introspect_db(dbname, conn=conn, reintrospection=True)\n\n def on_remote_system_config_change(self) -> None:\n if not self._accept_new_tasks:\n return\n\n # Triggered by a postgres notification event 'system-config-changes'\n # on the __edgedb_sysevent__ channel\n\n async def task():\n try:\n cfg = await self._load_sys_config()\n self._dbindex.update_sys_config(cfg)\n self._server.reinit_idle_gc_collector()\n except Exception:\n metrics.background_errors.inc(\n 1.0, self._instance_name, \"on_remote_system_config_change\"\n )\n raise\n\n self.create_task(task(), interruptable=True)\n\n def on_global_schema_change(self) -> None:\n if not self._accept_new_tasks:\n return\n\n async def task():\n try:\n await self._reintrospect_global_schema()\n except Exception:\n metrics.background_errors.inc(\n 1.0, self._instance_name, \"on_global_schema_change\"\n )\n raise\n\n self.create_task(task(), interruptable=True)\n\n async def _load_query_cache(\n self,\n conn: pgcon.PGConnection,\n keys: Optional[Iterable[uuid.UUID]] = None,\n ) -> list[tuple[bytes, ...]] | None:\n if keys is None:\n return await conn.sql_fetch(\n b'''\n SELECT \"schema_version\", \"input\", \"output\"\n FROM \"edgedb\".\"_query_cache\"\n ''',\n use_prep_stmt=True,\n )\n else:\n # If keys were specified, just load those keys.\n # TODO: Or should we do something time based?\n return await conn.sql_fetch(\n b'''\n SELECT \"schema_version\", \"input\", \"output\"\n ROWS FROM json_array_elements($1) j(ikey)\n INNER JOIN \"edgedb\".\"_query_cache\"\n ON (to_jsonb(ARRAY[ikey])->>0)::uuid = key\n ''',\n args=(json.dumps(keys).encode('utf-8'),),\n use_prep_stmt=True,\n )\n\n async def evict_query_cache(\n self,\n dbname: str,\n keys: Iterable[uuid.UUID],\n ) -> None:\n try:\n async with self._with_intro_pgcon(dbname) as conn:\n if not conn:\n return\n for key in keys:\n await conn.sql_fetch(\n b'SELECT \"edgedb\".\"_evict_query_cache\"($1)',\n args=(key.bytes,),\n use_prep_stmt=True,\n )\n\n except Exception:\n logger.exception(\"error in evict_query_cache():\")\n metrics.background_errors.inc(\n 1.0, self._instance_name, \"evict_query_cache\"\n )\n\n def on_remote_query_cache_change(\n self,\n dbname: str,\n to_add: Optional[list[str]],\n to_invalidate: Optional[list[str]],\n ) -> None:\n if not self.is_db_ready(dbname):\n return\n\n if to_invalidate:\n if db := self.maybe_get_db(dbname=dbname):\n db.invalidate_cache_entries(\n [uuid.UUID(s) for s in to_invalidate]\n )\n\n async def task():\n try:\n async with self._with_intro_pgcon(dbname) as conn:\n if not conn:\n return\n query_cache = await self._load_query_cache(\n conn, keys=to_add\n )\n\n if query_cache and (db := self.maybe_get_db(dbname=dbname)):\n db.hydrate_cache(query_cache)\n\n except Exception:\n logger.exception(\"error in on_remote_query_cache_change():\")\n metrics.background_errors.inc(\n 1.0, self._instance_name, \"on_remote_query_cache_change\"\n )\n raise\n\n # If neither to_add nor to_invalidate are specified, then we do\n # a full introspection.\n if to_add or not to_invalidate:\n self.create_task(task(), interruptable=True)\n\n def get_debug_info(self) -> dict[str, Any]:\n from . import smtp\n\n pgaddr = self.get_pgaddr()\n pgaddr.clear_server_settings()\n pgdict = pgaddr.__dict__\n del pgdict['database']\n pgaddr.__dict__ = pgdict\n\n obj = dict(\n params=dict(\n max_backend_connections=self._max_backend_connections,\n suggested_client_pool_size=self._suggested_client_pool_size,\n tenant_id=self._tenant_id,\n ),\n instance_config=config.debug_serialize_config(\n self.get_sys_config()),\n user_roles=self._roles,\n pg_addr=dict(\n server_settings=vars(self._cluster.get_connection_params()),\n dsn=pgaddr.to_dsn(),\n ),\n pg_pool=self._pg_pool._build_snapshot(now=time.monotonic()),\n )\n\n dbs = {}\n if self._dbindex is not None:\n for db in self._dbindex.iter_dbs():\n if db.name in defines.EDGEDB_SPECIAL_DBS:\n continue\n\n try:\n email_provider = dataclasses.asdict(\n smtp.get_current_email_provider(db)\n )\n except errors.ConfigurationError:\n email_provider = None\n dbs[db.name] = dict(\n name=db.name,\n dbver=db.dbver,\n config=(\n None\n if db.db_config is None\n else config.debug_serialize_config(db.db_config)\n ),\n extensions=sorted(db.extensions),\n query_cache_size=db.get_query_cache_size(),\n connections=[\n dict(\n in_tx=view.in_tx(),\n in_tx_error=view.in_tx_error(),\n config=config.debug_serialize_config(\n view.get_session_config()),\n module_aliases=view.get_modaliases(),\n )\n for view in db.iter_views()\n ],\n current_email_provider=email_provider,\n )\n\n obj[\"databases\"] = dbs\n\n return obj\n\n def get_compiler_args(self) -> dict[str, Any]:\n assert self._dbindex is not None\n return {\"dbindex\": self._dbindex}\n\n def iter_dbs(self) -> Iterator[dbview.Database]:\n if self._dbindex is not None:\n yield from self._dbindex.iter_dbs()\n\n\n# sentinel Tenant object to indicate an empty SNI\nhost_tenant = Tenant.__new__(Tenant)\n" }, { "path": "edb/testbase/__init__.py", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2020-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n" }, { "path": "edb/testbase/asyncutils.py", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2019-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\nimport unittest\n\ntry:\n import async_solipsism\nexcept ImportError:\n async_solipsism = None # type: ignore\n\n\ndef with_fake_event_loop(f):\n # async_solpsism creates an event loop with, among other things,\n # a totally fake clock which starts at 0.\n def new(*args, **kwargs):\n if not async_solipsism:\n raise unittest.SkipTest('async_solipsism is missing')\n\n loop = async_solipsism.EventLoop()\n try:\n loop.run_until_complete(f(*args, **kwargs))\n finally:\n loop.close()\n\n return new\n" }, { "path": "edb/testbase/cluster.py", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2016-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nfrom __future__ import annotations\nfrom typing import Any, Optional, Mapping, TYPE_CHECKING\n\nimport asyncio\nimport json\nimport os\nimport pathlib\nimport socket\nimport subprocess\nimport sys\nimport tempfile\nimport time\n\nfrom edb import buildmeta\nfrom edb.common import devmode\nfrom edb.edgeql import quote\n\nfrom edb.server import auth\nfrom edb.server import args as edgedb_args\nfrom edb.server import defines as edgedb_defines\nfrom edb.server import pgcluster\nfrom edb.server import pgconnparams\n\n\nif TYPE_CHECKING:\n from edb.server import pgcon\n\n\nclass ClusterError(Exception):\n pass\n\n\nclass BaseCluster:\n def __init__(\n self,\n runstate_dir: pathlib.Path,\n *,\n port: int = edgedb_defines.EDGEDB_PORT,\n env: Optional[Mapping[str, str]] = None,\n testmode: bool = False,\n log_level: Optional[str] = None,\n security: Optional[\n edgedb_args.ServerSecurityMode\n ] = None,\n http_endpoint_security: Optional[\n edgedb_args.ServerEndpointSecurityMode\n ] = None,\n compiler_pool_mode: Optional[\n edgedb_args.CompilerPoolMode\n ] = None,\n net_worker_mode: Optional[\n edgedb_args.NetWorkerMode\n ] = None,\n ):\n self._edgedb_cmd = [sys.executable, '-I', '-m', 'edb.server.main']\n\n if \"EDGEDB_SERVER_MULTITENANT_CONFIG_FILE\" not in os.environ:\n self._edgedb_cmd.append('--instance-name=localtest')\n\n self._edgedb_cmd.append('--tls-cert-mode=generate_self_signed')\n self._edgedb_cmd.append('--jose-key-mode=generate')\n\n if log_level:\n self._edgedb_cmd.extend(['--log-level', log_level])\n\n compiler_addr = os.getenv('EDGEDB_TEST_REMOTE_COMPILER')\n if compiler_addr:\n compiler_pool_mode = edgedb_args.CompilerPoolMode.Remote\n self._edgedb_cmd.extend(\n [\n '--compiler-pool-addr',\n compiler_addr,\n ]\n )\n\n if devmode.is_in_dev_mode():\n self._edgedb_cmd.append('--devmode')\n\n if testmode:\n self._edgedb_cmd.append('--testmode')\n\n if security:\n self._edgedb_cmd.extend((\n '--security',\n str(security),\n ))\n\n if http_endpoint_security:\n self._edgedb_cmd.extend((\n '--http-endpoint-security',\n str(http_endpoint_security),\n ))\n\n if compiler_pool_mode is not None:\n self._edgedb_cmd.extend((\n '--compiler-pool-mode',\n str(compiler_pool_mode),\n ))\n\n if net_worker_mode is not None:\n self._edgedb_cmd.extend((\n '--net-worker-mode',\n str(net_worker_mode),\n ))\n\n self._log_level = log_level\n self._runstate_dir = runstate_dir\n self._edgedb_cmd.extend(['--runstate-dir', str(runstate_dir)])\n self._pg_cluster: Optional[pgcluster.BaseCluster] = None\n self._pg_connect_args: pgconnparams.CreateParamsKwargs = {}\n self._daemon_process: Optional[subprocess.Popen[str]] = None\n self._port = port\n self._effective_port = None\n self._tls_cert_file = None\n self._env = env\n\n async def _get_pg_cluster(self) -> pgcluster.BaseCluster:\n if self._pg_cluster is None:\n self._pg_cluster = await self._new_pg_cluster()\n return self._pg_cluster\n\n async def _new_pg_cluster(self) -> pgcluster.BaseCluster:\n raise NotImplementedError()\n\n async def get_status(self) -> str:\n pg_cluster = await self._get_pg_cluster()\n pg_status = await pg_cluster.get_status()\n initially_stopped = pg_status == 'stopped'\n\n if initially_stopped:\n await pg_cluster.start()\n elif pg_status == 'not-initialized':\n return 'not-initialized'\n\n conn = None\n try:\n conn = await pg_cluster.connect(\n source_description=f\"{self.__class__.__name__}.get_status\",\n **self._pg_connect_args,\n )\n\n db_exists = await self._edgedb_template_exists(conn)\n finally:\n if conn is not None:\n conn.terminate()\n await asyncio.sleep(0)\n if initially_stopped:\n await pg_cluster.stop()\n\n if initially_stopped:\n return 'stopped' if db_exists else 'not-initialized,stopped'\n else:\n return 'running' if db_exists else 'not-initialized,running'\n\n def get_connect_args(self) -> dict[str, Any]:\n return {\n 'host': 'localhost',\n 'port': self._effective_port,\n 'tls_ca_file': self._tls_cert_file,\n }\n\n async def init(\n self,\n *,\n server_settings: Optional[Mapping[str, str]] = None,\n ) -> None:\n cluster_status = await self.get_status()\n\n if not cluster_status.startswith('not-initialized'):\n raise ClusterError('cluster has already been initialized')\n\n self._init()\n\n async def start(\n self,\n wait: int=60,\n *,\n port: Optional[int] = None,\n **settings: Any,\n ) -> None:\n if port is None:\n port = self._port\n\n if port == 0:\n cmd_port = 'auto'\n else:\n cmd_port = str(port)\n\n extra_args = ['--{}={}'.format(k.replace('_', '-'), v)\n for k, v in settings.items()]\n extra_args.append(f'--port={cmd_port}')\n status_r = status_w = None\n if port == 0:\n status_r, status_w = socket.socketpair()\n extra_args.append(f'--emit-server-status=fd://{status_w.fileno()}')\n\n env: Optional[dict[str, str]]\n if self._env:\n env = os.environ.copy()\n env.update(self._env)\n else:\n env = None\n\n self._daemon_process = subprocess.Popen(\n self._edgedb_cmd + extra_args,\n env=env,\n text=True,\n pass_fds=(status_w.fileno(),) if status_w is not None else (),\n )\n\n if status_w is not None:\n status_w.close()\n\n try:\n await self._wait_for_server(timeout=wait, status_sock=status_r)\n except Exception:\n self.stop()\n raise\n\n def stop(self, wait: int = 60) -> None:\n if (self._daemon_process is not None and\n self._daemon_process.returncode is None):\n self._daemon_process.terminate()\n self._daemon_process.wait(wait)\n\n def destroy(self) -> None:\n if self._pg_cluster is not None:\n self._pg_cluster.destroy()\n\n def _init(self) -> None:\n env: Optional[dict[str, str]]\n if self._env:\n env = os.environ.copy()\n env.update(self._env)\n else:\n env = None\n\n init = subprocess.run(\n self._edgedb_cmd + ['--bootstrap-only'],\n stdout=sys.stdout, stderr=sys.stderr,\n env=env)\n\n if init.returncode != 0:\n raise ClusterError(\n f'edgedb-server --bootstrap-only failed with '\n f'exit code {init.returncode}')\n\n async def _edgedb_template_exists(\n self,\n conn: pgcon.PGConnection,\n ) -> bool:\n return await conn.sql_fetch_val(\n b\"SELECT True FROM pg_catalog.pg_database WHERE datname = $1\",\n args=[edgedb_defines.EDGEDB_TEMPLATE_DB.encode(\"utf-8\")],\n ) is not None\n\n async def _wait_for_server(\n self,\n timeout: float = 30.0,\n status_sock: Optional[socket.socket] = None,\n ) -> None:\n\n async def _read_server_status(\n stream: asyncio.StreamReader,\n ) -> dict[str, Any]:\n while True:\n line = await stream.readline()\n if not line:\n raise ClusterError(\"Gel server terminated\")\n if line.startswith(b'READY='):\n break\n\n _, _, dataline = line.decode().partition('=')\n try:\n return json.loads(dataline) # type: ignore\n except Exception as e:\n raise ClusterError(\n f\"Gel server returned invalid status line: \"\n f\"{dataline!r} ({e})\"\n )\n\n async def test() -> None:\n stat_reader, stat_writer = await asyncio.open_connection(\n sock=status_sock,\n )\n try:\n data = await asyncio.wait_for(\n _read_server_status(stat_reader),\n timeout=timeout\n )\n except asyncio.TimeoutError:\n raise ClusterError(\n f'Gel server did not initialize '\n f'within {timeout} seconds'\n ) from None\n\n self._effective_port = data['port']\n self._tls_cert_file = data['tls_cert_file']\n stat_writer.close()\n\n left = timeout\n if status_sock is not None:\n started = time.monotonic()\n await test()\n left -= (time.monotonic() - started)\n if res := self._admin_query(\n \"SELECT ();\",\n f\"{max(1, int(left))}s\",\n check=False,\n ):\n raise ClusterError(\n f'could not connect to edgedb-server '\n f'within {timeout} seconds (exit code = {res})') from None\n\n def _admin_query(\n self,\n query: str,\n wait_until_available: str = \"0s\",\n check: bool=True,\n ) -> int:\n args = [\n \"gel\",\n \"query\",\n \"--unix-path\",\n str(os.path.abspath(self._runstate_dir)),\n \"--port\",\n str(self._effective_port),\n \"--admin\",\n \"--user\",\n edgedb_defines.EDGEDB_SUPERUSER,\n \"--branch\",\n edgedb_defines.EDGEDB_SUPERUSER_DB,\n \"--wait-until-available\",\n wait_until_available,\n query,\n ]\n res = subprocess.run(\n args=args,\n check=check,\n stdout=subprocess.PIPE,\n stderr=subprocess.STDOUT,\n )\n return res.returncode\n\n async def set_test_config(self) -> None:\n # Set session_idle_transaction_timeout to 5 minutes.\n self._admin_query('''\n CONFIGURE INSTANCE SET session_idle_transaction_timeout :=\n '5 minutes';\n ''')\n # And disable session_idle_timeout\n self._admin_query('''\n CONFIGURE INSTANCE SET session_idle_timeout :=\n '0 seconds';\n ''')\n\n async def set_superuser_password(self, password: str) -> None:\n self._admin_query(f'''\n ALTER ROLE {edgedb_defines.EDGEDB_SUPERUSER}\n SET password := {quote.quote_literal(password)}\n ''')\n\n async def trust_local_connections(self) -> None:\n self._admin_query('''\n CONFIGURE INSTANCE INSERT Auth {\n priority := 0,\n method := (INSERT Trust),\n }\n ''')\n\n def has_create_database(self) -> bool:\n return True\n\n def has_create_role(self) -> bool:\n return True\n\n\nclass Cluster(BaseCluster):\n def __init__(\n self,\n data_dir: pathlib.Path,\n *,\n pg_superuser: str = 'postgres',\n port: int = edgedb_defines.EDGEDB_PORT,\n runstate_dir: Optional[pathlib.Path] = None,\n env: Optional[Mapping[str, str]] = None,\n testmode: bool = False,\n log_level: Optional[str] = None,\n security: Optional[\n edgedb_args.ServerSecurityMode\n ] = None,\n http_endpoint_security: Optional[\n edgedb_args.ServerEndpointSecurityMode\n ] = None,\n compiler_pool_mode: Optional[\n edgedb_args.CompilerPoolMode\n ] = None,\n ) -> None:\n self._data_dir = data_dir\n if runstate_dir is None:\n runstate_dir = buildmeta.get_runstate_path(self._data_dir)\n super().__init__(\n runstate_dir,\n port=port,\n env=env,\n testmode=testmode,\n log_level=log_level,\n security=security,\n http_endpoint_security=http_endpoint_security,\n compiler_pool_mode=compiler_pool_mode,\n )\n self._edgedb_cmd.extend(['-D', str(self._data_dir)])\n self._pg_connect_args['user'] = pg_superuser\n self._pg_connect_args['database'] = 'template1'\n self._jws_key: Optional[auth.JWKSet] = None\n\n async def _new_pg_cluster(self) -> pgcluster.Cluster:\n return await pgcluster.get_local_pg_cluster(\n self._data_dir,\n runstate_dir=self._runstate_dir,\n log_level=self._log_level,\n )\n\n def get_data_dir(self) -> pathlib.Path:\n return self._data_dir\n\n async def init(\n self,\n *,\n server_settings: Optional[Mapping[str, str]] = None,\n ) -> None:\n cluster_status = await self.get_status()\n\n if not cluster_status.startswith('not-initialized'):\n raise ClusterError(\n 'cluster in {!r} has already been initialized'.format(\n self._data_dir))\n\n self._init()\n\n\nclass TempCluster(Cluster):\n def __init__(\n self,\n *,\n data_dir_suffix: Optional[str] = None,\n data_dir_prefix: Optional[str] = None,\n data_dir_parent: Optional[str] = None,\n env: Optional[Mapping[str, str]] = None,\n testmode: bool = False,\n log_level: Optional[str] = None,\n security: Optional[\n edgedb_args.ServerSecurityMode\n ] = None,\n http_endpoint_security: Optional[\n edgedb_args.ServerEndpointSecurityMode\n ] = None,\n compiler_pool_mode: Optional[\n edgedb_args.CompilerPoolMode\n ] = None,\n ) -> None:\n tempdir = pathlib.Path(\n tempfile.mkdtemp(\n suffix=data_dir_suffix,\n prefix=data_dir_prefix,\n dir=data_dir_parent,\n ),\n )\n super().__init__(\n data_dir=tempdir,\n runstate_dir=tempdir,\n env=env,\n testmode=testmode,\n log_level=log_level,\n security=security,\n http_endpoint_security=http_endpoint_security,\n compiler_pool_mode=compiler_pool_mode,\n )\n\n\nclass RunningCluster(BaseCluster):\n def __init__(self, **conn_args: Any) -> None:\n self.conn_args = conn_args\n\n def is_managed(self) -> bool:\n return False\n\n def ensure_initialized(self) -> bool:\n return False\n\n def get_connect_args(self) -> dict[str, Any]:\n return dict(self.conn_args)\n\n async def get_status(self) -> str:\n return 'running'\n\n async def init(\n self,\n *,\n server_settings: Optional[Mapping[str, str]] = None,\n ) -> None:\n pass\n\n async def start(\n self,\n wait: int=60,\n *,\n port: Optional[int] = None,\n **settings: Any,\n ) -> None:\n pass\n\n def stop(self, wait: int = 60) -> None:\n pass\n\n def destroy(self) -> None:\n pass\n\n def has_create_database(self) -> bool:\n return os.environ.get('EDGEDB_TEST_CASES_SET_UP') != 'inplace'\n\n def has_create_role(self) -> bool:\n return os.environ.get('EDGEDB_TEST_HAS_CREATE_ROLE') == 'True'\n\n\nclass TempClusterWithRemotePg(BaseCluster):\n def __init__(\n self,\n backend_dsn: str,\n *,\n data_dir_suffix: Optional[str] = None,\n data_dir_prefix: Optional[str] = None,\n data_dir_parent: Optional[str] = None,\n env: Optional[Mapping[str, str]] = None,\n testmode: bool = False,\n log_level: Optional[str] = None,\n security: Optional[\n edgedb_args.ServerSecurityMode\n ] = None,\n http_endpoint_security: Optional[\n edgedb_args.ServerEndpointSecurityMode\n ] = None,\n compiler_pool_mode: Optional[\n edgedb_args.CompilerPoolMode\n ] = None,\n ) -> None:\n runstate_dir = pathlib.Path(\n tempfile.mkdtemp(\n suffix=data_dir_suffix,\n prefix=data_dir_prefix,\n dir=data_dir_parent,\n ),\n )\n self._backend_dsn = backend_dsn\n mt = \"EDGEDB_SERVER_MULTITENANT_CONFIG_FILE\" in os.environ\n if mt:\n compiler_pool_mode = edgedb_args.CompilerPoolMode.MultiTenant\n\n super().__init__(\n runstate_dir,\n env=env,\n testmode=testmode,\n log_level=log_level,\n security=security,\n http_endpoint_security=http_endpoint_security,\n compiler_pool_mode=compiler_pool_mode,\n )\n if not mt:\n self._edgedb_cmd.extend(['--backend-dsn', backend_dsn])\n\n async def _new_pg_cluster(self) -> pgcluster.BaseCluster:\n return await pgcluster.get_remote_pg_cluster(self._backend_dsn)\n\n def has_create_database(self) -> bool:\n if self._pg_cluster:\n return self._pg_cluster.get_runtime_params().has_create_database\n else:\n return super().has_create_database()\n\n def has_create_role(self) -> bool:\n if self._pg_cluster:\n return self._pg_cluster.get_runtime_params().has_create_role\n else:\n return super().has_create_role()\n" }, { "path": "edb/testbase/connection.py", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2016-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\"\"\"A specialized client API for Gel tests.\n\nHistorically Gel tests relied on a very specific client API that\nis no longer supported by edgedb-python. Here we implement that API\n(for example, transactions can be nested and are non-retrying).\n\"\"\"\n\nfrom __future__ import annotations\nimport typing\n\nimport abc\nimport asyncio\nimport contextlib\nimport enum\nimport functools\nimport random\nimport socket\nimport ssl\nimport time\n\nfrom gel import abstract\nfrom gel import errors\nfrom gel import con_utils\nfrom gel import enums as edgedb_enums\nfrom gel import options\nfrom gel.protocol import protocol # type: ignore\n\nfrom edb.protocol import protocol as edb_protocol # type: ignore\n\n\nclass TransactionState(enum.Enum):\n NEW = 0\n STARTED = 1\n COMMITTED = 2\n ROLLEDBACK = 3\n FAILED = 4\n\n\nInputLanguage = protocol.InputLanguage\n\n\nclass BaseTransaction(abc.ABC):\n\n ID_COUNTER = 0\n\n def __init__(self, owner):\n self._connection = owner\n self._state = TransactionState.NEW\n self._managed = False\n self._nested = False\n\n type(self).ID_COUNTER += 1\n self._id = f'raw_tx_{self.ID_COUNTER}'\n\n def is_active(self) -> bool:\n return self._state is TransactionState.STARTED\n\n def __check_state_base(self, opname):\n if self._state is TransactionState.COMMITTED:\n raise errors.InterfaceError(\n f'cannot {opname}; the transaction is already committed')\n if self._state is TransactionState.ROLLEDBACK:\n raise errors.InterfaceError(\n f'cannot {opname}; the transaction is already rolled back')\n if self._state is TransactionState.FAILED:\n raise errors.InterfaceError(\n f'cannot {opname}; the transaction is in error state')\n\n def __check_state(self, opname):\n if self._state is not TransactionState.STARTED:\n if self._state is TransactionState.NEW:\n raise errors.InterfaceError(\n f'cannot {opname}; the transaction is not yet started')\n self.__check_state_base(opname)\n\n def _make_start_query(self):\n self.__check_state_base('start')\n if self._state is TransactionState.STARTED:\n raise errors.InterfaceError(\n 'cannot start; the transaction is already started')\n qry = self._make_start_query_inner()\n if self._connection._top_xact is None:\n self._connection._top_xact = self\n return qry\n\n @abc.abstractmethod\n def _make_start_query_inner(self):\n ...\n\n def _make_commit_query(self):\n self.__check_state('commit')\n\n if self._connection._top_xact is self:\n self._connection._top_xact = None\n\n return 'COMMIT;'\n\n def _make_rollback_query(self):\n self.__check_state('rollback')\n\n if self._connection._top_xact is self:\n self._connection._top_xact = None\n\n if self._nested:\n query = f'ROLLBACK TO SAVEPOINT {self._id};'\n else:\n query = 'ROLLBACK;'\n\n return query\n\n async def start(self) -> None:\n query = self._make_start_query()\n try:\n await self._connection.execute(query)\n except BaseException:\n self._state = TransactionState.FAILED\n raise\n else:\n self._state = TransactionState.STARTED\n\n async def commit(self) -> None:\n if self._managed:\n raise errors.InterfaceError(\n 'cannot manually commit from within an `async with` block')\n await self._commit()\n\n async def _commit(self) -> None:\n query = self._make_commit_query()\n try:\n # Use _fetchall to ensure there is no retry performed.\n # The protocol level apparently thinks the transaction is\n # over if COMMIT fails, and since we use that to decide\n # whether to retry in query/execute, it would want to\n # retry a COMMIT.\n await self._connection._fetchall(query)\n except BaseException:\n self._state = TransactionState.FAILED\n raise\n else:\n self._state = TransactionState.COMMITTED\n\n async def rollback(self) -> None:\n if self._managed:\n raise errors.InterfaceError(\n 'cannot manually rollback from within an `async with` block')\n await self._rollback()\n\n async def _rollback(self) -> None:\n query = self._make_rollback_query()\n try:\n await self._connection.execute(query)\n except BaseException:\n self._state = TransactionState.FAILED\n raise\n else:\n self._state = TransactionState.ROLLEDBACK\n\n\nclass RawTransaction(BaseTransaction):\n def _make_start_query_inner(self):\n con = self._connection\n\n if con._top_xact is not None:\n # Nested transaction block\n self._nested = True\n\n if self._nested:\n query = f'DECLARE SAVEPOINT {self._id};'\n else:\n query = 'START TRANSACTION;'\n\n return query\n\n def _make_commit_query(self):\n query = super()._make_commit_query()\n\n if self._nested:\n query = f'RELEASE SAVEPOINT {self._id};'\n\n return query\n\n def _make_rollback_query(self):\n query = super()._make_rollback_query()\n\n if self._nested:\n query = f'ROLLBACK TO SAVEPOINT {self._id};'\n\n return query\n\n async def __aenter__(self):\n if self._managed:\n raise errors.InterfaceError(\n 'cannot enter context: already in an `async with` block')\n self._managed = True\n await self.start()\n return self\n\n async def __aexit__(self, extype, ex, tb):\n try:\n if extype is not None:\n await self._rollback()\n else:\n await self._commit()\n finally:\n self._managed = False\n\n\nclass _Executor(abstract.AsyncIOExecutor):\n # TODO: Remove this, once we land this in gel-python and update\n # our bindings.\n async def query_graphql_json(\n self, query, *args: typing.Any, **kwargs: typing.Any\n ) -> str:\n return await self._query(\n abstract.QueryContext(\n query=abstract.QueryWithArgs(\n query,\n # None,\n args,\n kwargs,\n input_language=ord('G'),\n ),\n cache=self._get_query_cache(),\n query_options=abstract._query_single_json_opts,\n retry_options=self._get_retry_options(),\n state=self._get_state(),\n # transaction_options=self._get_active_tx_options(),\n warning_handler=self._get_warning_handler(),\n annotations=self._get_annotations(),\n )\n )\n\n\nclass Iteration(BaseTransaction, _Executor):\n def __init__(self, retry, connection, iteration):\n super().__init__(connection)\n self._options = retry._options.transaction_options\n self.__retry = retry\n self.__iteration = iteration\n self.__started = False\n\n async def __aenter__(self):\n if self._managed:\n raise errors.InterfaceError(\n 'cannot enter context: already in an `async with` block')\n self._managed = True\n return self\n\n async def __aexit__(self, extype, ex, tb):\n self._managed = False\n if not self.__started:\n return False\n\n try:\n if extype is None:\n await self._commit()\n else:\n await self._rollback()\n except errors.EdgeDBError as err:\n if ex is None:\n # On commit we don't know if commit is succeeded before the\n # database have received it or after it have been done but\n # network is dropped before we were able to receive a response\n raise err\n # If we were going to rollback, look at original error\n # to find out whether we want to retry, regardless of\n # the rollback error.\n # In this case we ignore rollback issue as original error is more\n # important, e.g. in case `CancelledError` it's important\n # to propagate it to cancel the whole task.\n # NOTE: rollback error is always swallowed, should we use\n # on_log_message for it?\n\n if (\n extype is not None and\n issubclass(extype, errors.EdgeDBError) and\n ex.has_tag(errors.SHOULD_RETRY)\n ):\n return self.__retry._retry(ex)\n\n def _make_start_query_inner(self):\n return self._options.start_transaction_query()\n\n def _get_query_cache(self) -> abstract.QueryCache:\n return self._connection._query_cache\n\n async def _query(self, query_context: abstract.QueryContext):\n await self._ensure_transaction()\n return await self._connection.raw_query(query_context)\n\n async def _execute(self, query: abstract.ExecuteContext) -> None:\n await self._ensure_transaction()\n await self._connection._execute(query)\n\n async def _ensure_transaction(self):\n if not self._managed:\n raise errors.InterfaceError(\n \"Only managed retriable transactions are supported. \"\n \"Use `async with transaction:`\"\n )\n if not self.__started:\n self.__started = True\n if self._connection.is_closed():\n await self._connection.connect(\n single_attempt=self.__iteration != 0\n )\n await self.start()\n\n def _get_retry_options(self) -> options.RetryOptions:\n return options.RetryOptions.defaults()\n\n def _get_state(self) -> options.State:\n return self._connection._get_state()\n\n def _get_warning_handler(self) -> options.WarningHandler:\n return self._connection._get_warning_handler()\n\n def _get_annotations(self) -> dict[str, str]:\n return self._connection._get_annotations()\n\n\nclass Retry:\n def __init__(self, connection, raw=False):\n self._connection = connection\n self._iteration = 0\n self._done = False\n self._next_backoff = 0\n self._options = connection._options\n\n def _retry(self, exc):\n self._last_exception = exc\n rule = self._options.retry_options.get_rule_for_exception(exc)\n if self._iteration >= rule.attempts:\n return False\n self._done = False\n self._next_backoff = rule.backoff(self._iteration)\n return True\n\n def __aiter__(self):\n return self\n\n async def __anext__(self):\n # Note: when changing this code consider also\n # updating Retry.__next__.\n if self._done:\n raise StopAsyncIteration\n if self._next_backoff:\n await asyncio.sleep(self._next_backoff)\n self._done = True\n iteration = Iteration(self, self._connection, self._iteration)\n self._iteration += 1\n return iteration\n\n\nclass Connection(options._OptionsMixin, _Executor):\n\n _top_xact: RawTransaction | None = None\n\n def __init__(\n self, connect_args, *, test_no_tls=False, server_hostname=None\n ):\n super().__init__()\n self._connect_args = connect_args\n self._protocol = None\n self._transport = None\n self._query_cache = abstract.QueryCache(\n codecs_registry=protocol.CodecsRegistry(),\n query_cache=protocol.LRUMapping(maxsize=1000),\n )\n self._test_no_tls = test_no_tls\n self._params = None\n self._server_hostname = server_hostname\n self._log_listeners = set()\n self._capture_warnings = None\n\n def add_log_listener(self, callback):\n self._log_listeners.add(callback)\n\n def remove_log_listener(self, callback):\n self._log_listeners.discard(callback)\n\n def _get_retry_options(self) -> options.RetryOptions:\n return self._options.retry_options\n\n def _get_state(self):\n return self._options.state\n\n def _get_annotations(self) -> dict[str, str]:\n return self._options.annotations\n\n @contextlib.contextmanager\n def capture_warnings(self) -> typing.Iterator[list[errors.EdgeDBError]]:\n old = self._capture_warnings\n warnings: list[errors.EdgeDBError] = []\n self._capture_warnings = warnings\n try:\n yield warnings\n finally:\n self._capture_warnings = old\n\n def _warning_handler(self, warnings, res):\n if self._capture_warnings is not None:\n self._capture_warnings.extend(warnings)\n return res\n else:\n raise warnings[0]\n\n def _get_warning_handler(self) -> options.WarningHandler:\n return self._warning_handler\n\n def _on_log_message(self, msg):\n if self._log_listeners:\n loop = asyncio.get_running_loop()\n for cb in self._log_listeners:\n loop.call_soon(cb, self, msg)\n\n def _shallow_clone(self):\n con = self.__class__.__new__(self.__class__)\n con._connect_args = self._connect_args\n con._protocol = self._protocol\n con._query_cache = self._query_cache\n con._test_no_tls = self._test_no_tls\n con._params = self._params\n con._server_hostname = self._server_hostname\n return con\n\n def _get_query_cache(self) -> abstract.QueryCache:\n return self._query_cache\n\n async def ensure_connected(self):\n if self.is_closed():\n await self.connect()\n return self\n\n async def _query(self, query_context: abstract.QueryContext):\n await self.ensure_connected()\n return await self.raw_query(query_context)\n\n async def _retry_operation(self, func):\n i = 0\n while True:\n i += 1\n try:\n return await func()\n # Retry transaction conflict errors, up to a maximum of 5\n # times. We don't do this if we are in a transaction,\n # since that *ought* to be done at the transaction level.\n # Though in reality in the test suite it is usually done at the\n # test runner level.\n except errors.TransactionConflictError:\n if i >= 10 or self.is_in_transaction():\n raise\n await asyncio.sleep(\n min((2 ** i) * 0.1, 10)\n + random.randrange(100) * 0.001\n )\n\n def _prohibit_state(self, state) -> None:\n # The testbase connection uses our own subclass of\n # gel-python's AsyncIOProtocol,\n # edb.protocol.protocol.Protocol, that overrides encode_state\n # to ignore any user specified state and to always just use\n # whatever the server suggests.\n #\n # It is probably possible to make CONFIGURE .../SET GLOBAL\n # play nicely with with_globals/etc, by decoding what the server\n # has sent and then overlaying the user configured stuff.\n #\n # Since it doesn't work, disable it.\n if state.as_dict():\n raise AssertionError(\n f'test suite client cannot use with_XXX config methods; '\n f'use SET ... in the protocol instead '\n f'or use the stock python client '\n f'(or go make it work; that would be nice too)\\n'\n f'config was: {state.as_dict()}'\n )\n\n async def _execute(self, script: abstract.ExecuteContext) -> None:\n await self.ensure_connected()\n self._prohibit_state(script.state)\n\n async def _inner():\n ctx = script.lower(allow_capabilities=edgedb_enums.Capability.ALL)\n res = await self._protocol.execute(ctx)\n if ctx.warnings:\n script.warning_handler(ctx.warnings, res)\n\n await self._retry_operation(_inner)\n\n async def raw_query(self, query_context: abstract.QueryContext):\n self._prohibit_state(query_context.state)\n\n async def _inner():\n ctx = query_context.lower(\n allow_capabilities=edgedb_enums.Capability.ALL)\n res = await self._protocol.query(ctx)\n if ctx.warnings:\n res = query_context.warning_handler(ctx.warnings, res)\n return res\n\n return await self._retry_operation(_inner)\n\n async def _fetchall_generic(self, ctx):\n await self.ensure_connected()\n res = await self._protocol.query(ctx)\n if ctx.warnings:\n res = self._get_warning_handler()(ctx.warnings, res)\n return res\n\n async def _fetchall(\n self,\n query: str,\n *args,\n __language__: protocol.InputLanguage = protocol.InputLanguage.EDGEQL,\n __limit__: int = 0,\n __typeids__: bool = False,\n __typenames__: bool = False,\n __allow_capabilities__: edgedb_enums.Capability = (\n edgedb_enums.Capability.ALL),\n **kwargs,\n ):\n return await self._fetchall_generic(\n protocol.ExecuteContext(\n query=query,\n args=args,\n kwargs=kwargs,\n reg=self._query_cache.codecs_registry,\n qc=self._query_cache.query_cache,\n implicit_limit=__limit__,\n inline_typeids=__typeids__,\n inline_typenames=__typenames__,\n input_language=__language__,\n output_format=protocol.OutputFormat.BINARY,\n allow_capabilities=__allow_capabilities__,\n )\n )\n\n async def _fetchall_json(\n self,\n query: str,\n *args,\n __limit__: int = 0,\n **kwargs,\n ):\n return await self._fetchall_generic(\n protocol.ExecuteContext(\n query=query,\n args=args,\n kwargs=kwargs,\n reg=self._query_cache.codecs_registry,\n qc=self._query_cache.query_cache,\n implicit_limit=__limit__,\n inline_typenames=False,\n input_language=protocol.InputLanguage.EDGEQL,\n output_format=protocol.OutputFormat.JSON,\n )\n )\n\n async def _fetchall_json_elements(self, query: str, *args, **kwargs):\n return await self._fetchall_generic(\n protocol.ExecuteContext(\n query=query,\n args=args,\n kwargs=kwargs,\n reg=self._query_cache.codecs_registry,\n qc=self._query_cache.query_cache,\n input_language=protocol.InputLanguage.EDGEQL,\n output_format=protocol.OutputFormat.JSON_ELEMENTS,\n allow_capabilities=edgedb_enums.Capability.EXECUTE, # type: ignore\n )\n )\n\n def _clear_codecs_cache(self):\n self._query_cache.codecs_registry.clear_cache()\n\n def _get_last_status(self) -> typing.Optional[str]:\n if self._protocol is None:\n return None\n status = self._protocol.last_status\n if status is not None:\n status = status.decode()\n return status\n\n def _get_last_capabilities(\n self,\n ) -> typing.Optional[edgedb_enums.Capability]:\n if self._protocol is None:\n return None\n else:\n return self._protocol.last_capabilities\n\n def is_closed(self):\n return self._protocol is None or not self._protocol.connected\n\n async def connect(self, single_attempt=False):\n self._params, client_config = con_utils.parse_connect_arguments(\n **self._connect_args,\n tls_server_name=None,\n command_timeout=None,\n server_settings=None,\n )\n start = time.monotonic()\n if single_attempt:\n max_time = 0\n else:\n max_time = start + client_config.wait_until_available\n iteration = 1\n\n while True:\n addr = self._params.address\n try:\n await asyncio.wait_for(\n self.connect_addr(),\n client_config.connect_timeout,\n )\n except TimeoutError as e:\n if iteration > 1 and time.monotonic() >= max_time:\n raise errors.ClientConnectionTimeoutError(\n f\"connecting to {addr} failed in\"\n f\" {client_config.connect_timeout} sec\"\n ) from e\n except errors.ClientConnectionError as e:\n if (\n not e.has_tag(errors.SHOULD_RECONNECT) or\n (iteration > 1 and time.monotonic() >= max_time)\n ):\n nice_err = e.__class__(\n con_utils.render_client_no_connection_error(\n e,\n addr,\n attempts=iteration,\n duration=time.monotonic() - start,\n ))\n raise nice_err from e.__cause__\n else:\n return\n\n iteration += 1\n await asyncio.sleep(0.01 + random.random() * 0.2)\n\n async def connect_addr(self):\n tr = None\n loop = asyncio.get_running_loop()\n addr = self._params.address\n protocol_factory = functools.partial(\n edb_protocol.Protocol, self._params, loop\n )\n\n try:\n if isinstance(addr, str):\n # UNIX socket\n tr, pr = await loop.create_unix_connection(\n protocol_factory, addr\n )\n elif self._test_no_tls:\n tr, pr = await loop.create_connection(protocol_factory, *addr)\n else:\n try:\n tr, pr = await loop.create_connection(\n protocol_factory,\n *addr,\n server_hostname=self._server_hostname,\n ssl=self._params.ssl_ctx,\n )\n except ssl.CertificateError as e:\n raise con_utils.wrap_error(e) from e\n except ssl.SSLError as e:\n if e.reason == 'CERTIFICATE_VERIFY_FAILED':\n raise con_utils.wrap_error(e) from e\n tr, pr = await loop.create_connection(\n protocol_factory,\n *addr,\n )\n else:\n con_utils.check_alpn_protocol(\n tr.get_extra_info('ssl_object')\n )\n except socket.gaierror as e:\n # All name resolution errors are considered temporary\n raise errors.ClientConnectionFailedTemporarilyError(str(e)) from e\n except OSError as e:\n raise con_utils.wrap_error(e) from e\n except Exception:\n if tr is not None:\n tr.close()\n raise\n\n pr.set_connection(self)\n\n try:\n await pr.connect()\n except OSError as e:\n if tr is not None:\n tr.close()\n raise con_utils.wrap_error(e) from e\n except BaseException:\n if tr is not None:\n tr.close()\n raise\n\n self._protocol = pr\n self._transport = tr\n\n def retrying_transaction(self) -> Retry:\n return Retry(self)\n\n def transaction(self) -> RawTransaction:\n return RawTransaction(self)\n\n def is_in_transaction(self):\n return self._protocol.is_in_transaction()\n\n def get_settings(self) -> dict[str, typing.Any]:\n return self._protocol.get_settings()\n\n @property\n def dbname(self) -> str:\n return self._params.database\n\n def connected_addr(self):\n return self._params.address\n\n async def aclose(self):\n if not self.is_closed():\n try:\n self._protocol.terminate()\n await self._protocol.wait_for_disconnect()\n except (Exception, asyncio.CancelledError):\n self.terminate()\n raise\n\n def terminate(self):\n if not self.is_closed():\n self._protocol.abort()\n\n def get_transport(self):\n return self._transport\n\n\nasync def async_connect_test_client(\n dsn: typing.Optional[str] = None,\n host: typing.Optional[str] = None,\n port: typing.Optional[int] = None,\n credentials: typing.Optional[str] = None,\n credentials_file: typing.Optional[str] = None,\n user: typing.Optional[str] = None,\n password: typing.Optional[str] = None,\n secret_key: typing.Optional[str] = None,\n branch: typing.Optional[str] = None,\n database: typing.Optional[str] = None,\n tls_ca: typing.Optional[str] = None,\n tls_ca_file: typing.Optional[str] = None,\n tls_security: typing.Optional[str] = None,\n test_no_tls: bool = False,\n wait_until_available: int = 30,\n timeout: int = 10,\n server_hostname: str | None = None,\n) -> Connection:\n return await Connection(\n {\n \"dsn\": dsn,\n \"host\": host,\n \"port\": port,\n \"credentials\": credentials,\n \"credentials_file\": credentials_file,\n \"user\": user,\n \"password\": password,\n \"secret_key\": secret_key,\n \"branch\": branch,\n \"database\": database,\n \"timeout\": timeout,\n \"tls_ca\": tls_ca,\n \"tls_ca_file\": tls_ca_file,\n \"tls_security\": tls_security,\n \"wait_until_available\": wait_until_available,\n },\n test_no_tls=test_no_tls,\n server_hostname=server_hostname,\n ).ensure_connected()\n" }, { "path": "edb/testbase/experimental_interpreter.py", "content": "\nfrom __future__ import annotations\nfrom typing import (\n Any,\n Optional,\n)\nimport unittest\n\n\nfrom edb.common import assert_data_shape\nfrom edb.tools.experimental_interpreter.new_interpreter import EdgeQLInterpreter\n\nbag = assert_data_shape.bag\n\n\nclass ExperimentalInterpreterTestCase(unittest.TestCase):\n SCHEMA: Optional[str] = None\n SETUP: Optional[str] = None\n INTERPRETER_USE_SQLITE = False\n\n client: EdgeQLInterpreter\n initial_state: object\n\n @classmethod\n def setUpClass(cls):\n if cls.SCHEMA is not None:\n with open(cls.SCHEMA) as f:\n schema_content = f.read()\n else:\n schema_content = \"\"\n\n sqlite_filename = None\n if cls.INTERPRETER_USE_SQLITE:\n sqlite_filename = \":memory:\"\n\n try:\n import sqlite3\n except ModuleNotFoundError:\n raise unittest.SkipTest(\"sqlite is not installed\")\n\n if sqlite3.sqlite_version_info < (3, 37):\n raise unittest.SkipTest(\"sqlite version is too old (need 3.37)\")\n\n cls.client = EdgeQLInterpreter(schema_content, sqlite_filename)\n\n if cls.SETUP is not None:\n with open(cls.SETUP) as f:\n setup_content = f.read()\n cls.client.query_str(setup_content)\n\n cls.initial_state = cls.client.db.dump_state()\n\n def setUp(self):\n self.client.db.restore_state(self.initial_state)\n\n def execute(self, query: str, *, variables=None) -> Any:\n return self.client.run_single_str_get_json_with_cache(\n query, variables=variables)\n\n def execute_single(self, query: str, *, variables=None) -> Any:\n return self.client.query_single_json(query, variables=variables)\n\n def assert_query_result(self, query,\n exp_result_json,\n exp_result_binary=...,\n *,\n msg: Optional[str] = None,\n sort: Optional[bool] = None,\n variables=None,\n ):\n if (hasattr(self, \"use_experimental_interpreter\") and\n self.use_experimental_interpreter):\n result = self.client.run_single_str_get_json_with_cache(\n query, variables=variables)\n res = result\n if sort is not None:\n assert_data_shape.sort_results(res, sort)\n if exp_result_binary is not ...:\n assert_data_shape.assert_data_shape(\n res, exp_result_binary, self.fail, message=msg)\n else:\n assert_data_shape.assert_data_shape(\n res, exp_result_json, self.fail, message=msg)\n" }, { "path": "edb/testbase/http.py", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2019-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nfrom __future__ import annotations\nfrom typing import (\n Any,\n Callable,\n Optional,\n)\n\nimport asyncio\nimport http.server\nimport json\nimport threading\nimport urllib.parse\nimport urllib.request\nimport dataclasses\nimport time\nimport random\n\nimport gel\n\nfrom edb.errors import base as base_errors\n\nfrom edb.common import assert_data_shape\n\nfrom . import server\n\n\nbag = assert_data_shape.bag\n\n\nclass BaseHttpTest(server.QueryTestCase):\n @classmethod\n async def _wait_for_db_config(\n cls,\n config_key,\n *,\n server=None,\n instance_config=False,\n value=None,\n is_reset=False,\n ):\n dbname = cls.get_database_name()\n # Wait for the database config changes to propagate to the\n # server by watching a debug endpoint\n async for tr in cls.try_until_succeeds(\n ignore=AssertionError,\n timeout=120,\n ):\n async with tr:\n with cls.http_con(server) as http_con:\n (\n rdata,\n _headers,\n _status,\n ) = cls.http_con_request(\n http_con,\n prefix=\"\",\n path=\"server-info\",\n )\n data = json.loads(rdata)\n if \"databases\" not in data:\n # multi-tenant instance - use the first tenant\n data = next(iter(data[\"tenants\"].values()))\n if instance_config:\n config = data[\"instance_config\"]\n else:\n config = data[\"databases\"][dbname][\"config\"]\n if is_reset:\n if config_key in config:\n raise AssertionError(\"database config not ready\")\n else:\n if config_key not in config:\n raise AssertionError(\"database config not ready\")\n if value and config[config_key] != value:\n raise AssertionError(\"database config not ready\")\n\n\nclass BaseHttpExtensionTest(BaseHttpTest):\n @classmethod\n def get_extension_path(cls):\n raise NotImplementedError\n\n @classmethod\n def get_api_prefix(cls):\n extpath = cls.get_extension_path()\n dbname = cls.get_database_name()\n return f\"/branch/{dbname}/{extpath}\"\n\n\nclass ExtAuthTestCase(BaseHttpExtensionTest):\n EXTENSIONS = [\"pgcrypto\", \"auth\"]\n\n @classmethod\n def get_extension_path(cls):\n return \"ext/auth\"\n\n def generate_pkce_pair(self) -> tuple[str, str]:\n \"\"\"Generate a PKCE verifier and its corresponding challenge.\n\n Returns:\n (verifier, challenge): tuple of str\n \"\"\"\n import os\n import base64\n import hashlib\n\n verifier = base64.urlsafe_b64encode(os.urandom(43)).rstrip(b'=')\n challenge = base64.urlsafe_b64encode(\n hashlib.sha256(verifier).digest()\n ).rstrip(b'=')\n return verifier.decode(), challenge.decode()\n\n\nclass EdgeQLTestCase(BaseHttpExtensionTest):\n EXTENSIONS = [\"edgeql_http\"]\n\n @classmethod\n def get_extension_path(cls):\n return \"edgeql\"\n\n def edgeql_query(\n self,\n query,\n *,\n use_http_post=True,\n variables=None,\n globals=None,\n config=None,\n origin=None,\n\n user=None,\n password=None,\n ):\n req_data = {\"query\": query}\n\n if use_http_post:\n if variables is not None:\n req_data[\"variables\"] = variables\n if globals is not None:\n req_data[\"globals\"] = globals\n if config is not None:\n req_data[\"config\"] = config\n req = urllib.request.Request(self.http_addr, method=\"POST\")\n req.add_header(\"Content-Type\", \"application/json\")\n req.add_header(\n \"Authorization\", self.make_auth_header(user, password)\n )\n if origin:\n req.add_header(\"Origin\", origin)\n response = urllib.request.urlopen(\n req, json.dumps(req_data).encode(), context=self.tls_context\n )\n resp_data = json.loads(response.read())\n else:\n if variables is not None:\n req_data[\"variables\"] = json.dumps(variables)\n if globals is not None:\n req_data[\"globals\"] = json.dumps(globals)\n if config is not None:\n req_data[\"config\"] = json.dumps(config)\n req = urllib.request.Request(\n f\"{self.http_addr}/?{urllib.parse.urlencode(req_data)}\",\n )\n req.add_header(\n \"Authorization\", self.make_auth_header(user, password)\n )\n response = urllib.request.urlopen(\n req,\n context=self.tls_context,\n )\n resp_data = json.loads(response.read())\n\n if \"data\" in resp_data:\n return (resp_data[\"data\"], response)\n\n err = resp_data[\"error\"]\n\n ex_msg = err[\"message\"].strip()\n ex_code = err[\"code\"]\n\n raise gel.EdgeDBError._from_code(ex_code, ex_msg)\n\n def assert_edgeql_query_result(\n self,\n query,\n result,\n *,\n msg=None,\n sort=None,\n use_http_post=True,\n variables=None,\n globals=None,\n config=None,\n ):\n res, _ = self.edgeql_query(\n query,\n use_http_post=use_http_post,\n variables=variables,\n globals=globals,\n config=config,\n )\n\n if sort is not None:\n # GQL will always have a single object returned. The data is\n # in the top-level fields, so that's what needs to be sorted.\n for r in res.values():\n assert_data_shape.sort_results(r, sort)\n\n assert_data_shape.assert_data_shape(res, result, self.fail, message=msg)\n return res\n\n\nclass GraphQLTestCase(BaseHttpExtensionTest):\n EXTENSIONS = [\"graphql\"]\n\n @classmethod\n def get_extension_path(cls):\n return \"graphql\"\n\n def graphql_query(\n self,\n query,\n *,\n operation_name=None,\n use_http_post=True,\n variables=None,\n globals=None,\n deprecated_globals=None,\n config=None,\n\n user=None,\n password=None,\n ):\n def inner():\n return self._graphql_query(\n query,\n operation_name=operation_name,\n use_http_post=use_http_post,\n variables=variables,\n globals=globals,\n deprecated_globals=deprecated_globals,\n config=config,\n user=user,\n password=password,\n )\n return self._retry_operation(inner)\n\n def _retry_operation(self, func):\n i = 0\n while True:\n i += 1\n try:\n return func()\n\n # Retry transaction conflict errors\n except gel.errors.TransactionConflictError:\n if i >= 10:\n raise\n time.sleep(\n min((2 ** i) * 0.1, 10)\n + random.randrange(100) * 0.001\n )\n\n def _graphql_query(\n self,\n query,\n *,\n operation_name=None,\n use_http_post=True,\n variables=None,\n globals=None,\n deprecated_globals=None,\n config=None,\n user=None,\n password=None,\n ):\n req_data = {\"query\": query}\n\n if operation_name is not None:\n req_data[\"operationName\"] = operation_name\n\n if use_http_post:\n if variables is not None:\n req_data[\"variables\"] = variables\n if globals is not None:\n if variables is None:\n req_data[\"variables\"] = dict()\n req_data[\"variables\"][\"__globals__\"] = globals\n if config is not None:\n if variables is None:\n req_data[\"variables\"] = dict()\n req_data[\"variables\"][\"__config__\"] = config\n # Support testing the old way of sending globals.\n if deprecated_globals is not None:\n req_data[\"globals\"] = deprecated_globals\n\n req = urllib.request.Request(self.http_addr, method=\"POST\")\n req.add_header(\"Content-Type\", \"application/json\")\n req.add_header(\n \"Authorization\", self.make_auth_header(user, password)\n )\n response = urllib.request.urlopen(\n req, json.dumps(req_data).encode(), context=self.tls_context\n )\n resp_data = json.loads(response.read())\n else:\n if globals is not None:\n if variables is None:\n variables = dict()\n variables[\"__globals__\"] = globals\n if config is not None:\n if variables is None:\n variables = dict()\n variables[\"__config__\"] = config\n # Support testing the old way of sending globals.\n if deprecated_globals is not None:\n req_data[\"globals\"] = json.dumps(deprecated_globals)\n if variables is not None:\n req_data[\"variables\"] = json.dumps(variables)\n req = urllib.request.Request(\n f\"{self.http_addr}/?{urllib.parse.urlencode(req_data)}\",\n )\n req.add_header(\n \"Authorization\", self.make_auth_header(user, password)\n )\n response = urllib.request.urlopen(\n req,\n context=self.tls_context,\n )\n resp_data = json.loads(response.read())\n\n if \"data\" in resp_data:\n return resp_data[\"data\"]\n\n err = resp_data[\"errors\"][0]\n\n typename, msg = err[\"message\"].split(\":\", 1)\n msg = msg.strip()\n\n try:\n ex_type = getattr(gel, typename)\n except AttributeError:\n raise AssertionError(\n f\"server returned an invalid exception typename: {typename!r}\"\n f\"\\n Message: {msg}\"\n )\n\n ex = ex_type(msg)\n\n if \"locations\" in err:\n # XXX Fix this when LSP \"location\" objects are implemented\n ex._attrs[base_errors.FIELD_LINE_START] = str(\n err[\"locations\"][0][\"line\"]\n ).encode()\n ex._attrs[base_errors.FIELD_COLUMN_START] = str(\n err[\"locations\"][0][\"column\"]\n ).encode()\n\n raise ex\n\n async def _native_graphql_query(\n self,\n query,\n *,\n # Can/should we support operation_name somehow...\n variables=None,\n globals=None,\n ):\n # The graphql tests are all synchronous, and our gel\n # connections need to be async... so we spin up a new\n # connection and asyncio.run the coro.\n con = await self.connect()\n try:\n # Ahhhhhh. We don't support with_globals on testbase\n # connections, so....\n if globals:\n glob_defs = {\n obj.name: obj\n for obj in\n await con.query('''\n select schema::Global {\n name, required, tname := .target.name\n }\n ''')\n }\n\n for k, v in globals.items():\n glob = glob_defs[k]\n # Why do we allow this for the HTTP proto??\n # We don't for binary proto stuff.\n if v is None and glob.required:\n continue\n mod = 'required ' if glob.required else ''\n await con.execute(\n f'set global {k} := <{mod}{glob.tname}>$0',\n json.dumps(v),\n )\n\n async with server.RollbackChanges(con):\n return json.loads(await con.query_graphql_json(\n query,\n **(variables or {}),\n ))\n finally:\n await con.aclose()\n\n def assert_graphql_query_result(\n self,\n query,\n result,\n *,\n msg=None,\n sort=None,\n operation_name=None,\n use_http_post=True,\n native_variables=None,\n variables=None,\n globals=None,\n deprecated_globals=None,\n config=None,\n ):\n # Try to use the native protocol first!\n if operation_name is None and config is None:\n try:\n res = asyncio.run(self._native_graphql_query(\n query,\n variables=(\n native_variables if native_variables is not None\n else variables\n ),\n globals=globals or deprecated_globals,\n ))\n\n if sort is not None:\n # GQL will always have a single object\n # returned. The data is in the top-level fields,\n # so that's what needs to be sorted.\n for r in res.values():\n assert_data_shape.sort_results(r, sort)\n\n assert_data_shape.assert_data_shape(\n res, result, self.fail, message=msg)\n except gel.UnsupportedFeatureError as e:\n if 'Default variables are not supported' in str(e):\n # Whatever.\n pass\n else:\n raise\n\n res = self.graphql_query(\n query,\n operation_name=operation_name,\n use_http_post=use_http_post,\n variables=variables,\n globals=globals,\n deprecated_globals=deprecated_globals,\n config=config,\n )\n\n if sort is not None:\n # GQL will always have a single object returned. The data is\n # in the top-level fields, so that's what needs to be sorted.\n for r in res.values():\n assert_data_shape.sort_results(r, sort)\n\n assert_data_shape.assert_data_shape(res, result, self.fail, message=msg)\n return res\n\n\nclass MockHttpServerHandler(http.server.BaseHTTPRequestHandler):\n def get_server_and_path(self) -> tuple[str, str]:\n server = f'http://{self.headers.get(\"Host\")}'\n return server, self.path\n\n def do_GET(self):\n self.close_connection = False\n server, path = self.get_server_and_path()\n self.server.owner.handle_request(\"GET\", server, path, self)\n\n def do_POST(self):\n self.close_connection = False\n server, path = self.get_server_and_path()\n self.server.owner.handle_request(\"POST\", server, path, self)\n\n def log_message(self, *args):\n pass\n\n\nclass MultiHostMockHttpServerHandler(MockHttpServerHandler):\n def get_server_and_path(self) -> tuple[str, str]:\n # Path looks like:\n # http://127.0.0.1:32881/https%3A//slack.com/.well-known/openid-configuration\n raw_url = urllib.parse.unquote(self.path.lstrip(\"/\"))\n url = urllib.parse.urlparse(raw_url)\n return (f\"{url.scheme}://{url.netloc}\", url.path.lstrip(\"/\"))\n\n\nResponseType = tuple[str, int] | tuple[str, int, dict[str, str]]\n\n\n@dataclasses.dataclass\nclass RequestDetails:\n headers: dict[str, str | Any]\n query_params: dict[str, list[str]]\n body: Optional[str]\n\n\nclass MockHttpServer:\n def __init__(\n self,\n handler_type: type[MockHttpServerHandler] = MockHttpServerHandler,\n port: int = 0,\n ) -> None:\n self._port = port\n self.has_started = threading.Event()\n self.routes: dict[\n tuple[str, str, str],\n (\n ResponseType\n | Callable[\n [MockHttpServerHandler, RequestDetails], ResponseType\n ]\n ),\n ] = {}\n self.requests: dict[tuple[str, str, str], list[RequestDetails]] = {}\n self.url: Optional[str] = None\n self.handler_type = handler_type\n\n def get_base_url(self) -> str:\n if self.url is None:\n raise RuntimeError(\"mock server is not running\")\n return self.url\n\n def register_route_handler(\n self,\n method: str,\n server: str,\n path: str,\n ):\n def wrapper(\n handler: (\n ResponseType\n | Callable[\n [MockHttpServerHandler, RequestDetails], ResponseType\n ]\n ),\n ):\n self.routes[(method, server, path)] = handler\n return handler\n\n return wrapper\n\n def handle_request(\n self,\n method: str,\n server: str,\n path: str,\n handler: MockHttpServerHandler,\n ):\n # `handler` is documented here:\n # https://docs.python.org/3/library/http.server.html#http.server.BaseHTTPRequestHandler\n key = (method, server, path)\n if key not in self.requests:\n self.requests[key] = []\n\n # Parse and save the request details\n parsed_path = urllib.parse.urlparse(path)\n headers = {k.lower(): v for k, v in dict(handler.headers).items()}\n query_params = urllib.parse.parse_qs(parsed_path.query)\n if \"content-length\" in headers:\n body = handler.rfile.read(int(headers[\"content-length\"])).decode()\n else:\n body = None\n\n request_details = RequestDetails(\n headers=headers,\n query_params=query_params,\n body=body,\n )\n self.requests[key].append(request_details)\n if key not in self.routes:\n error_message = (\n f\"No route handler for {key}\\n\\n\"\n f\"Available routes:\\n{self.routes}\"\n )\n handler.send_error(404, message=error_message)\n return\n\n registered_handler = self.routes[key]\n\n if callable(registered_handler):\n try:\n handler_result = registered_handler(handler, request_details)\n if len(handler_result) == 2:\n response, status = handler_result\n additional_headers = None\n elif len(handler_result) == 3:\n response, status, additional_headers = handler_result\n except Exception:\n handler.send_error(500)\n raise\n else:\n if len(registered_handler) == 2:\n response, status = registered_handler\n additional_headers = None\n elif len(registered_handler) == 3:\n response, status, additional_headers = registered_handler\n\n accept_header = request_details.headers.get(\n \"accept\", \"application/json\"\n )\n\n if (\n accept_header.startswith(\"application/json\")\n or (\n accept_header.startswith(\"application/\")\n and \"vnd.\" in accept_header\n and \"+json\" in accept_header\n )\n or accept_header == \"*/*\"\n ):\n content_type = \"application/json\"\n elif accept_header.startswith(\"application/x-www-form-urlencoded\"):\n content_type = \"application/x-www-form-urlencoded\"\n else:\n handler.send_error(\n 415, f\"Unsupported accept header: {accept_header}\"\n )\n return\n\n data = response.encode()\n\n handler.send_response(status)\n handler.send_header(\"Content-Type\", content_type)\n handler.send_header(\"Content-Length\", str(len(data)))\n if additional_headers is not None:\n for header, value in additional_headers.items():\n handler.send_header(header, value)\n handler.end_headers()\n handler.wfile.write(data)\n\n def start(self):\n assert not hasattr(self, \"_http_runner\")\n self._http_runner = threading.Thread(target=self._http_worker)\n self._http_runner.start()\n self.has_started.wait()\n self.url = f\"http://{self._address[0]}:{self._address[1]}/\"\n\n def __enter__(self):\n self.start()\n return self\n\n def _http_worker(self):\n self._http_server = http.server.HTTPServer(\n (\"localhost\", self._port), self.handler_type\n )\n self._http_server.owner = self\n self._address = self._http_server.server_address\n self.has_started.set()\n self._http_server.serve_forever(poll_interval=0.01)\n self._http_server.server_close()\n\n def stop(self):\n self._http_server.shutdown()\n if self._http_runner is not None:\n self._http_runner.join(timeout=60)\n if self._http_runner.is_alive():\n raise RuntimeError(\"Mock HTTP server failed to stop\")\n self._http_runner = None\n\n def __exit__(self, *exc):\n self.stop()\n self.url = None\n" }, { "path": "edb/testbase/lang.py", "content": "# mypy: ignore-errors\n\n#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2016-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nfrom __future__ import annotations\nfrom typing import Any, Optional\n\nimport typing\nimport functools\nimport os\nimport re\nimport unittest\n\nfrom edb.common import span\nfrom edb.common import debug\nfrom edb.common import devmode\nfrom edb.common import markup\n\nfrom edb import buildmeta\nfrom edb import errors\nfrom edb import edgeql\nfrom edb.edgeql import ast as qlast\nfrom edb.edgeql import parser as qlparser\nfrom edb.edgeql.parser import grammar as qlgrammar\nfrom edb.edgeql import qltypes\n\nfrom edb.server import compiler as edbcompiler\n\nfrom edb.schema import ddl as s_ddl\nfrom edb.schema import delta as sd\nfrom edb.schema import migrations as s_migrations # noqa\nfrom edb.schema import reflection as s_refl\nfrom edb.schema import schema as s_schema\nfrom edb.schema import std as s_std\nfrom edb.schema import utils as s_utils\nfrom edb.schema import modules as s_mod\n\n\ndef must_fail(exc_type, exc_msg_re=None, **kwargs):\n \"\"\"A decorator to ensure that the test fails with a specific exception.\n\n If exc_msg_re is passed, assertRaisesRegex will be used to match the\n exception message.\n\n Example:\n\n @must_fail(EdgeQLSyntaxError,\n 'non-default argument follows', line=2, col=61)\n def test_edgeql_syntax_1(self):\n ...\n \"\"\"\n def wrap(func):\n args = (exc_type,)\n if exc_msg_re is not None:\n args += (exc_msg_re,)\n\n _set_spec(func, 'must_fail', (args, kwargs))\n return func\n return wrap\n\n\ndef _set_spec(func, name, attrs):\n try:\n spec = func.test_spec\n except AttributeError:\n spec = func.test_spec = {}\n\n assert name not in spec\n spec[name] = attrs\n\n\nclass DocTestMeta(type(unittest.TestCase)):\n def __new__(mcls, name, bases, dct):\n for attr, meth in tuple(dct.items()):\n if attr.startswith('test_') and meth.__doc__:\n\n @functools.wraps(meth)\n def wrapper(self, meth=meth, doc=meth.__doc__):\n spec = getattr(meth, 'test_spec', {})\n spec['test_name'] = meth.__name__\n\n if doc:\n output = error = None\n\n source, _, output = doc.partition('\\n% OK %')\n\n if not output:\n source, _, error = doc.partition('\\n% ERROR %')\n\n if not error:\n output = None\n else:\n output = error\n else:\n source = output = None\n\n self._run_test(source=source, spec=spec, expected=output)\n\n dct[attr] = wrapper\n\n return super().__new__(mcls, name, bases, dct)\n\n\nclass BaseDocTest(unittest.TestCase, metaclass=DocTestMeta):\n parser_debug_flag = ''\n re_filter: Optional[typing.Pattern[str]] = None\n\n def _run_test(self, *, source, spec=None, expected=None):\n if spec and 'must_fail' in spec:\n spec_args, spec_kwargs = spec['must_fail']\n\n if len(spec_args) == 1:\n assertRaises = self.assertRaises\n else:\n assertRaises = self.assertRaisesRegex\n\n with assertRaises(*spec_args) as cm:\n return self.run_test(source=source, spec=spec,\n expected=expected)\n\n if cm.exception:\n exc = cm.exception\n for attr_name, expected_val in spec_kwargs.items():\n val = getattr(exc, attr_name)\n if val != expected_val:\n raise AssertionError(\n f'must_fail: attribute {attr_name!r} is '\n f'{val} (expected is {expected_val!r})') from exc\n else:\n return self.run_test(source=source, spec=spec, expected=expected)\n\n def run_test(self, *, source, spec, expected=None):\n raise NotImplementedError\n\n def assert_equal(\n self,\n expected,\n result,\n *,\n re_filter: Optional[str] = None,\n message: Optional[str] = None\n ) -> None:\n if re_filter is None:\n re_filter = self.re_filter\n\n if re_filter is not None:\n expected_stripped = re_filter.sub('', expected).lower()\n result_stripped = re_filter.sub('', result).lower()\n else:\n expected_stripped = expected.lower()\n result_stripped = result.lower()\n\n self.assertEqual(\n expected_stripped,\n result_stripped,\n (f'{message if message else \"\"}' +\n f'\\nexpected:\\n{expected}\\nreturned:\\n{result}')\n )\n\n\nclass BaseSyntaxTest(BaseDocTest):\n ast_to_source: Optional[Any] = None\n markup_dump_lexer: Optional[str] = None\n\n @classmethod\n def get_grammar_token(cls) -> type[qlgrammar.tokens.GrammarToken]:\n raise NotImplementedError\n\n def run_test(self, *, source, spec, expected=None):\n debug = bool(os.environ.get(self.parser_debug_flag))\n if debug:\n markup.dump_code(source, lexer=self.markup_dump_lexer)\n\n inast = qlparser.parse(self.get_grammar_token(), source)\n\n if debug:\n markup.dump(inast)\n\n # make sure that the AST has context\n span.SpanValidator().visit(inast)\n\n processed_src = self.ast_to_source(inast)\n\n if debug:\n markup.dump_code(processed_src, lexer=self.markup_dump_lexer)\n\n expected_src = source if expected is None else expected\n\n self.assert_equal(expected_src, processed_src)\n\n\n_std_schema = None\n_refl_schema = None\n_schema_class_layout = None\n\n\ndef _load_std_schema():\n global _std_schema\n if _std_schema is None:\n std_dirs_hash = buildmeta.hash_dirs(s_std.CACHE_SRC_DIRS)\n schema = None\n\n if devmode.is_in_dev_mode():\n schema = buildmeta.read_data_cache(\n std_dirs_hash, 'transient-stdschema.pickle')\n\n if schema is None:\n schema = s_schema.EMPTY_SCHEMA\n for modname in [*s_schema.STD_SOURCES, *s_schema.TESTMODE_SOURCES]:\n schema = s_std.load_std_module(schema, modname)\n schema, _ = s_std.make_schema_version(schema)\n schema, _ = s_std.make_global_schema_version(schema)\n\n if devmode.is_in_dev_mode():\n buildmeta.write_data_cache(\n schema, std_dirs_hash, 'transient-stdschema.pickle')\n\n _std_schema = schema\n\n return _std_schema\n\n\ndef _load_reflection_schema():\n global _refl_schema\n global _schema_class_layout\n\n if _refl_schema is None:\n std_dirs_hash = buildmeta.hash_dirs(s_std.CACHE_SRC_DIRS)\n\n cache = None\n if devmode.is_in_dev_mode():\n cache = buildmeta.read_data_cache(\n std_dirs_hash, 'transient-reflschema.pickle')\n\n if cache is not None:\n reflschema, classlayout = cache\n else:\n std_schema = _load_std_schema()\n reflection = s_refl.generate_structure(std_schema)\n classlayout = reflection.class_layout\n context = sd.CommandContext(stdmode=True)\n reflschema = reflection.intro_schema_delta.apply(\n std_schema, context)\n\n if devmode.is_in_dev_mode():\n buildmeta.write_data_cache(\n (reflschema, classlayout),\n std_dirs_hash,\n 'transient-reflschema.pickle',\n )\n\n _refl_schema = reflschema\n _schema_class_layout = classlayout\n\n return _refl_schema, _schema_class_layout\n\n\ndef new_compiler():\n std_schema = _load_std_schema()\n refl_schema, layout = _load_reflection_schema()\n\n return edbcompiler.new_compiler(\n std_schema=std_schema,\n reflection_schema=refl_schema,\n schema_class_layout=layout,\n )\n\n\nclass BaseSchemaTest(BaseDocTest):\n DEFAULT_MODULE = 'default'\n SCHEMA: Optional[str] = None\n\n schema: s_schema.Schema\n\n @classmethod\n def setUpClass(cls):\n script = cls.get_schema_script()\n if script is not None:\n cls.schema = cls.run_ddl(_load_std_schema(), script)\n else:\n cls.schema = _load_std_schema()\n\n @classmethod\n def run_ddl(cls, schema, ddl, default_module=s_mod.DEFAULT_MODULE_ALIAS):\n statements = edgeql.parse_block(ddl)\n\n current_schema = schema\n target_schema = None\n migration_schema = None\n migration_target = None\n migration_script = []\n\n for stmt in statements:\n if isinstance(stmt, qlast.StartMigration):\n # START MIGRATION\n if target_schema is None:\n target_schema = _load_std_schema()\n\n migration_target, _ = s_ddl.apply_sdl(\n stmt.target,\n base_schema=target_schema,\n testmode=True,\n )\n\n migration_schema = current_schema\n\n ddl_plan = None\n\n elif isinstance(stmt, qlast.PopulateMigration):\n # POPULATE MIGRATION\n if migration_target is None:\n raise errors.QueryError(\n 'unexpected POPULATE MIGRATION:'\n ' not currently in a migration block',\n span=stmt.span,\n )\n\n migration_diff = s_ddl.delta_schemas(\n migration_schema,\n migration_target,\n )\n\n if debug.flags.delta_plan:\n debug.header('Populate Migration Diff')\n debug.dump(migration_diff, schema=schema)\n\n new_ddl = s_ddl.ddlast_from_delta(\n migration_schema,\n migration_target,\n migration_diff,\n )\n\n migration_script.extend(new_ddl)\n\n if debug.flags.delta_plan:\n debug.header('Populate Migration DDL AST')\n text = []\n for cmd in new_ddl:\n debug.dump(cmd)\n text.append(edgeql.generate_source(cmd, pretty=True))\n debug.header('Populate Migration DDL Text')\n debug.dump_code(';\\n'.join(text) + ';')\n\n elif isinstance(stmt, qlast.DescribeCurrentMigration):\n # This is silly, and we don't bother doing all the work,\n # but try to catch when doing the JSON thing wouldn't work.\n if stmt.language is qltypes.DescribeLanguage.JSON:\n guided_diff = s_ddl.delta_schemas(\n migration_schema,\n migration_target,\n generate_prompts=True,\n )\n s_ddl.statements_from_delta(\n schema,\n migration_target,\n guided_diff,\n )\n\n elif isinstance(stmt, qlast.CommitMigration):\n if migration_target is None:\n raise errors.QueryError(\n 'unexpected COMMIT MIGRATION:'\n ' not currently in a migration block',\n span=stmt.span,\n )\n\n last_migration = current_schema.get_last_migration()\n if last_migration:\n last_migration_ref = s_utils.name_to_ast_ref(\n last_migration.get_name(current_schema),\n )\n else:\n last_migration_ref = None\n\n create_migration = qlast.CreateMigration(\n body=qlast.NestedQLBlock(commands=migration_script),\n parent=last_migration_ref,\n )\n\n ddl_plan = s_ddl.delta_from_ddl(\n create_migration,\n schema=migration_schema,\n modaliases={None: default_module},\n testmode=True,\n )\n\n if debug.flags.delta_plan:\n debug.header('Delta Plan')\n debug.dump(ddl_plan, schema=schema)\n\n migration_schema = None\n migration_target = None\n migration_script = []\n\n elif isinstance(stmt, qlast.DDLCommand):\n if migration_target is not None:\n migration_script.append(stmt)\n ddl_plan = None\n else:\n ddl_plan = s_ddl.delta_from_ddl(\n stmt,\n schema=current_schema,\n modaliases={None: default_module},\n testmode=True,\n )\n\n if debug.flags.delta_plan:\n debug.header('Delta Plan')\n debug.dump(ddl_plan, schema=schema)\n else:\n raise ValueError(\n f'unexpected {stmt!r} in compiler setup script')\n\n if ddl_plan is not None:\n context = sd.CommandContext()\n context.testmode = True\n current_schema = ddl_plan.apply(current_schema, context)\n\n return current_schema\n\n @classmethod\n def load_schema(\n cls, source: str, modname: Optional[str] = None\n ) -> s_schema.Schema:\n if not modname:\n modname = cls.DEFAULT_MODULE\n sdl_schema = qlparser.parse_sdl(f'module {modname} {{ {source} }}')\n schema = _load_std_schema()\n return s_ddl.apply_sdl(\n sdl_schema,\n base_schema=schema,\n )[0]\n\n @classmethod\n def get_schema_script(cls):\n script = ''\n\n # look at all SCHEMA entries and potentially create multiple modules\n schema = []\n for name in dir(cls):\n val = getattr(cls, name)\n m = re.match(r'^SCHEMA(?:_(\\w+))?', name)\n if m and val:\n module_name = (m.group(1)\n or 'default').lower().replace('_', '::')\n\n if '\\n' in val:\n # Inline schema source\n module = val\n else:\n with open(val, 'r') as sf:\n module = sf.read()\n\n schema.append(f'\\nmodule {module_name} {{ {module} }}')\n\n if schema:\n script += f'\\nSTART MIGRATION'\n script += f' TO {{ {\"\".join(schema)} }};'\n script += f'\\nPOPULATE MIGRATION;'\n script += f'\\nCOMMIT MIGRATION;'\n\n return script.strip(' \\n')\n\n\nclass BaseSchemaLoadTest(BaseSchemaTest):\n def run_test(self, *, source, spec, expected=None):\n self.load_schema(source)\n\n\nclass BaseEdgeQLCompilerTest(BaseSchemaTest):\n @classmethod\n def get_schema_script(cls):\n script = super().get_schema_script()\n if not script:\n raise ValueError(\n 'compiler test cases must define at least one '\n 'schema in the SCHEMA[_MODNAME] class attribute.')\n return script\n" }, { "path": "edb/testbase/proc.py", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2021-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\nimport asyncio\nimport socket\nimport sys\nimport unittest\n\nfrom edb.common import devmode\nfrom . import server\n\nexec(sys.argv[1], globals(), locals())\n\n\nclass ProcTest(server.TestCase):\n def notify_parent(self, mark):\n self.parent_writer.write(str(mark).encode() + b\"\\n\")\n\n async def wait_for_parent(self, mark):\n self.assertEqual(\n (await self.parent_reader.readline()).strip(),\n str(mark).encode(),\n )\n\n @classmethod\n def setUpClass(cls):\n super().setUpClass()\n\n async def _setup():\n sock = socket.fromfd(\n int(sys.argv[3]), socket.AF_UNIX, socket.SOCK_STREAM\n )\n cls.parent_reader, cls.parent_writer = (\n await asyncio.open_connection(sock=sock)\n )\n cls.loop.run_until_complete(_setup())\n\n exec(sys.argv[2], globals(), locals())\n\n\ndef main():\n cov_config = devmode.CoverageConfig.from_environ()\n if cov_config:\n cov = cov_config.new_coverage_object()\n cov.start()\n try:\n unittest.main(argv=sys.argv[:1], verbosity=2)\n finally:\n cov.stop()\n cov.save()\n else:\n unittest.main(argv=sys.argv[:1], verbosity=2)\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "edb/testbase/protocol/__init__.py", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2020-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n" }, { "path": "edb/testbase/protocol/test.py", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2020-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nfrom __future__ import annotations\n\nfrom edb.testbase import server\n\nfrom edb.protocol import protocol # type: ignore\nfrom edb.protocol.protocol import Connection\n\n\nclass ProtocolTestCase(server.DatabaseTestCase):\n\n PARALLELISM_GRANULARITY = 'database'\n BASE_TEST_CLASS = True\n\n con: Connection\n\n def setUp(self):\n self.con = self.loop.run_until_complete(\n protocol.new_connection(\n **self.get_connect_args(database=self.get_database_name())\n )\n )\n\n def tearDown(self):\n try:\n self.loop.run_until_complete(\n self.con.aclose()\n )\n finally:\n self.con = None\n" }, { "path": "edb/testbase/serutils.py", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2019-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nfrom __future__ import annotations\n\nimport dataclasses\nimport datetime\nimport decimal\nimport functools\nimport uuid\n\nimport edgedb\n\n\n@functools.singledispatch\ndef serialize(o):\n raise TypeError(f'cannot serialize type {type(o)}')\n\n\n@serialize.register\ndef _tuple(o: edgedb.Tuple):\n return [serialize(el) for el in o]\n\n\n@serialize.register\ndef _namedtuple(o: edgedb.NamedTuple):\n return {attr: serialize(getattr(o, attr)) for attr in dir(o)}\n\n\n@serialize.register\ndef _object(o: edgedb.Object):\n # We iterate over dataclasses.fields(o) (instead of dir(o))\n # because it contains both regular pointers and link properties,\n # and is I think the only current way to extract the names of all\n # the link properties\n attrs = [field.name for field in dataclasses.fields(o)]\n return {attr: serialize(getattr(o, attr)) for attr in attrs}\n\n\n@serialize.register(edgedb.Set)\n@serialize.register(edgedb.Array)\ndef _set(o):\n return [serialize(el) for el in o]\n\n\n@serialize.register(uuid.UUID)\ndef _stringify(o):\n return str(o)\n\n\n@serialize.register(int)\n@serialize.register(float)\n@serialize.register(str)\n@serialize.register(bytes)\n@serialize.register(bool)\n@serialize.register(type(None))\n@serialize.register(decimal.Decimal)\n@serialize.register(datetime.timedelta)\n@serialize.register(edgedb.RelativeDuration)\n@serialize.register(edgedb.DateDuration)\ndef _scalar(o):\n return o\n\n\n@serialize.register\ndef _datetime(o: datetime.datetime):\n return o.isoformat()\n\n\n@serialize.register\ndef _date(o: datetime.date):\n return o.isoformat()\n\n\n@serialize.register\ndef _time(o: datetime.time):\n return o.isoformat()\n\n\n@serialize.register\ndef _enum(o: edgedb.EnumValue):\n return str(o)\n\n\n@serialize.register\ndef _record(o: edgedb.Record):\n return {k: serialize(v) for k, v in o.as_dict().items()}\n\n\n@serialize.register\ndef _range(o: edgedb.Range):\n return {\n 'lower': serialize(o.lower),\n 'inc_lower': o.inc_lower,\n 'upper': serialize(o.upper),\n 'inc_upper': o.inc_upper,\n 'empty': o.is_empty(),\n }\n\n\n@serialize.register\ndef _multirane(o: edgedb.MultiRange):\n return [serialize(el) for el in o]\n\n\n@serialize.register\ndef _cfg_memory(o: edgedb.ConfigMemory):\n return str(o)\n" }, { "path": "edb/testbase/server.py", "content": "# mypy: ignore-errors\n\n#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2016-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nfrom __future__ import annotations\nfrom typing import (\n Any,\n Optional,\n Iterable,\n Literal,\n Sequence,\n NamedTuple,\n TYPE_CHECKING,\n)\nimport typing\n\nimport asyncio\nimport atexit\nimport base64\nimport contextlib\nimport dataclasses\nimport functools\nimport heapq\nimport http\nimport http.client\nimport inspect\nimport json\nimport os\nimport pathlib\nimport random\nimport re\nimport secrets\nimport shlex\nimport socket\nimport ssl\nimport struct\nimport subprocess\nimport sys\nimport tempfile\nimport time\nimport unittest\nimport urllib\n\nimport edgedb\n\nfrom edb.edgeql import quote as qlquote\nfrom edb.server import args as edgedb_args\nfrom edb.testbase import cluster as edgedb_cluster\nfrom edb.server import pgcluster\nfrom edb.server import defines as edgedb_defines\nfrom edb.server import auth\nfrom edb.server.pgconnparams import ConnectionParams\n\nfrom edb.common import assert_data_shape\nfrom edb.common import devmode\nfrom edb.common import debug\nfrom edb.common import retryloop\nfrom edb.common import secretkey\n\nfrom edb import buildmeta\nfrom edb import protocol\nfrom edb.protocol import protocol as test_protocol\nfrom edb.testbase import serutils\n\nfrom edb.testbase import connection as tconn\n\n\nif TYPE_CHECKING:\n import asyncpg\n DatabaseName = str\n SetupScript = str\n\n\ndef _add_test(result, test):\n # test is a tuple of the same test method that may zREPEAT\n cls = type(test[0])\n try:\n methods, repeat_methods = result[cls]\n except KeyError:\n # put zREPEAT tests in a separate list\n methods = []\n repeat_methods = []\n result[cls] = methods, repeat_methods\n\n methods.append(test[0])\n if len(test) > 1:\n repeat_methods.extend(test[1:])\n\n\ndef _merge_results(result):\n # make sure all the zREPEAT tests comes in the end\n return {k: v[0] + v[1] for k, v in result.items()}\n\n\ndef _get_test_cases(tests):\n result = {}\n\n for test in tests:\n if isinstance(test, unittest.TestSuite):\n result.update(_get_test_cases(test._tests))\n elif not getattr(test, '__unittest_skip__', False):\n _add_test(result, (test,))\n\n return result\n\n\ndef get_test_cases(tests):\n return _merge_results(_get_test_cases(tests))\n\n\nbag = assert_data_shape.bag\n\ngenerate_jwk = auth.generate_jwk\ngenerate_tls_cert = secretkey.generate_tls_cert\n\n\nclass CustomSNI_HTTPSConnection(http.client.HTTPSConnection):\n def __init__(self, *args, server_hostname=..., **kwargs):\n super().__init__(*args, **kwargs)\n self.server_hostname = server_hostname\n\n def connect(self):\n super(http.client.HTTPSConnection, self).connect()\n\n if self._tunnel_host:\n server_hostname = self._tunnel_host\n elif self.server_hostname is not ...:\n server_hostname = self.server_hostname\n else:\n server_hostname = self.host\n\n self.sock = self._context.wrap_socket(self.sock,\n server_hostname=server_hostname)\n\n def true_close(self):\n self.close()\n\n\nclass StubbornHttpConnection(CustomSNI_HTTPSConnection):\n\n def close(self):\n # Don't actually close the connection. This allows us to\n # test keep-alive and \"Connection: close\" headers.\n pass\n\n def true_close(self):\n http.client.HTTPConnection.close(self)\n\n\nclass TestCaseMeta(type(unittest.TestCase)):\n _database_names = set()\n\n @staticmethod\n def _iter_methods(bases, ns):\n for base in bases:\n for methname in dir(base):\n if not methname.startswith('test_'):\n continue\n\n meth = getattr(base, methname)\n if not inspect.iscoroutinefunction(meth):\n continue\n\n yield methname, meth\n\n for methname, meth in ns.items():\n if not methname.startswith('test_'):\n continue\n\n if not inspect.iscoroutinefunction(meth):\n continue\n\n yield methname, meth\n\n @classmethod\n def wrap(mcls, meth, is_repeat=False):\n @functools.wraps(meth)\n def wrapper(self, *args, __meth__=meth, **kwargs):\n try_no = 1\n\n if is_repeat and not getattr(self, 'TRANSACTION_ISOLATION', False):\n raise unittest.SkipTest()\n\n self.is_repeat = is_repeat\n while True:\n try:\n # There might be unobvious serializability\n # anomalies across the test suite, so, rather\n # than hunting them down every time, simply\n # retry the test.\n self.loop.run_until_complete(\n __meth__(self, *args, **kwargs))\n except (edgedb.TransactionSerializationError,\n edgedb.TransactionDeadlockError):\n if (\n try_no == 10\n # Only do a retry loop when we have a transaction\n or not getattr(self, 'TRANSACTION_ISOLATION', False)\n ):\n raise\n else:\n self.loop.run_until_complete(self.xact.rollback())\n self.loop.run_until_complete(asyncio.sleep(\n min((2 ** try_no) * 0.1, 10)\n + random.randrange(100) * 0.001\n ))\n self.xact = self.con.transaction()\n self.loop.run_until_complete(self.xact.start())\n\n try_no += 1\n else:\n break\n\n return wrapper\n\n @classmethod\n def add_method(mcls, methname, ns, meth):\n ns[methname] = mcls.wrap(meth)\n\n # If EDGEDB_TEST_REPEATS is set, duplicate all the tests.\n # This is valuable because it should exercise the function\n # cache.\n if (\n os.environ.get('EDGEDB_TEST_REPEATS', None)\n and methname.startswith('test_')\n ):\n new = methname.replace('test_', 'test_zREPEAT_', 1)\n ns[new] = mcls.wrap(meth, is_repeat=True)\n\n def __new__(mcls, name, bases, ns):\n for methname, meth in mcls._iter_methods(bases, ns.copy()):\n if methname in ns:\n del ns[methname]\n mcls.add_method(methname, ns, meth)\n\n cls = super().__new__(mcls, name, bases, ns)\n if not ns.get('BASE_TEST_CLASS') and hasattr(cls, 'get_database_name'):\n dbname = cls.get_database_name()\n\n if name in mcls._database_names:\n raise TypeError(\n f'{name} wants duplicate database name: {dbname}')\n\n mcls._database_names.add(name)\n\n return cls\n\n\nclass TestCase(unittest.TestCase, metaclass=TestCaseMeta):\n is_repeat: bool = False\n\n @classmethod\n def setUpClass(cls):\n loop = asyncio.new_event_loop()\n asyncio.set_event_loop(loop)\n cls.loop = loop\n\n @classmethod\n def tearDownClass(cls):\n cls.loop.close()\n asyncio.set_event_loop(None)\n\n @classmethod\n def uses_server(cls) -> bool:\n return True\n\n def add_fail_notes(self, **kwargs):\n if getattr(self, 'fail_notes', None) is None:\n self.fail_notes = {}\n self.fail_notes.update(kwargs)\n\n @contextlib.contextmanager\n def annotate(self, **kwargs):\n # Annotate the test in case the nested block of code fails.\n try:\n yield\n except Exception:\n self.add_fail_notes(**kwargs)\n raise\n\n @contextlib.contextmanager\n def assertRaisesRegex(self, exception, regex, msg=None, **kwargs):\n with super().assertRaisesRegex(exception, regex, msg=msg):\n try:\n yield\n except BaseException as e:\n if isinstance(e, exception):\n for attr_name, expected_val in kwargs.items():\n val = getattr(e, attr_name)\n if val != expected_val:\n raise self.failureException(\n f'{exception.__name__} context attribute '\n f'{attr_name!r} is {val} (expected '\n f'{expected_val!r})') from e\n raise\n\n @staticmethod\n def try_until_succeeds(\n *,\n ignore: type[Exception] | tuple[type[Exception]] | None = None,\n ignore_regexp: str | None = None,\n delay: float=0.5,\n timeout: float=5\n ):\n \"\"\"Retry a block of code a few times ignoring the specified errors.\n\n Example:\n\n async for tr in self.try_until_succeeds(\n ignore=edgedb.AuthenticationError):\n async with tr:\n await edgedb.connect(...)\n\n \"\"\"\n if ignore is None and ignore_regexp is None:\n raise ValueError('Expect at least one of ignore or ignore_regexp')\n return retryloop.RetryLoop(\n backoff=retryloop.const_backoff(delay),\n timeout=timeout,\n ignore=ignore,\n ignore_regexp=ignore_regexp,\n )\n\n @staticmethod\n def try_until_fails(\n *,\n wait_for: type[Exception] | tuple[type[Exception]] | None = None,\n wait_for_regexp: str | None = None,\n delay: float=0.5,\n timeout: float=5\n ):\n \"\"\"Retry a block of code a few times until the specified error happens.\n\n Example:\n\n async for tr in self.try_until_fails(\n wait_for=edgedb.AuthenticationError):\n async with tr:\n await edgedb.connect(...)\n\n \"\"\"\n if wait_for is None and wait_for_regexp is None:\n raise ValueError(\n 'Expect at least one of wait_for or wait_for_regexp'\n )\n return retryloop.RetryLoop(\n backoff=retryloop.const_backoff(delay),\n timeout=timeout,\n wait_for=wait_for,\n wait_for_regexp=wait_for_regexp,\n )\n\n def addCleanup(self, func, *args, **kwargs):\n @functools.wraps(func)\n def cleanup():\n res = func(*args, **kwargs)\n if inspect.isawaitable(res):\n self.loop.run_until_complete(res)\n super().addCleanup(cleanup)\n\n def __getstate__(self):\n # TestCases get pickled when run in in separate OS processes\n # via `edb test -jN`. If they reference any unpickleable objects,\n # the test engine crashes with no indication why and on what test.\n # That said, most of the TestCases' guts are not needed for the\n # test results renderer, so we only keep the essential attributes\n # here.\n\n outcome = self._outcome\n if outcome is not None and getattr(outcome, \"errors\", []):\n # We don't use `test._outcome` to render errors in\n # our renderers.\n outcome.errors = []\n\n return {\n '_testMethodName': self._testMethodName,\n '_outcome': outcome,\n '_testMethodDoc': self._testMethodDoc,\n '_subtest': self._subtest,\n '_cleanups': [],\n '_type_equality_funcs': self._type_equality_funcs,\n 'fail_notes': getattr(self, 'fail_notes', None),\n }\n\n @contextlib.contextmanager\n def assertChange(\n self, measure_fn: typing.Callable[[], int | float],\n expected_change: int | float\n ):\n before = measure_fn()\n try:\n yield\n finally:\n after = measure_fn()\n change = after - before\n self.assertEqual(expected_change, change)\n\n\nclass RollbackException(Exception):\n pass\n\n\nclass RollbackChanges:\n def __init__(self, con):\n self._conn = con\n\n async def __aenter__(self):\n self._tx = self._conn.transaction()\n await self._tx.start()\n\n async def __aexit__(self, exc_type, exc, tb):\n await self._tx.rollback()\n\n\nclass TestCaseWithHttpClient(TestCase):\n @classmethod\n def get_api_prefix(cls):\n return ''\n\n @classmethod\n @contextlib.contextmanager\n def http_con(\n cls,\n server,\n keep_alive=True,\n client_cert_file=None,\n client_key_file=None,\n **kwargs,\n ):\n conn_args = server.get_connect_args()\n tls_context = ssl.create_default_context(\n ssl.Purpose.SERVER_AUTH,\n cafile=conn_args[\"tls_ca_file\"],\n )\n tls_context.check_hostname = False\n if any((client_cert_file, client_key_file)):\n tls_context.load_cert_chain(client_cert_file, client_key_file)\n if keep_alive:\n ConCls = StubbornHttpConnection\n else:\n ConCls = CustomSNI_HTTPSConnection\n\n con = ConCls(\n conn_args[\"host\"],\n conn_args[\"port\"],\n context=tls_context,\n **kwargs,\n )\n con.connect()\n try:\n yield con\n finally:\n con.true_close()\n\n @classmethod\n def http_con_send_request(\n cls,\n http_con: http.client.HTTPConnection,\n params: Optional[dict[str, str]] = None,\n *,\n prefix: Optional[str] = None,\n headers: Optional[dict[str, str]] = None,\n method: str = \"GET\",\n body: bytes = b\"\",\n path: str = \"\",\n ):\n url = f'https://{http_con.host}:{http_con.port}'\n if prefix is None:\n prefix = cls.get_api_prefix()\n if prefix:\n url = f'{url}{prefix}'\n if path:\n url = f'{url}/{path}'\n if params is not None:\n url = f'{url}?{urllib.parse.urlencode(params)}'\n if headers is None:\n headers = {}\n http_con.request(method, url, body=body, headers=headers)\n\n @classmethod\n def http_con_read_response(\n cls,\n http_con: http.client.HTTPConnection,\n ) -> tuple[bytes, dict[str, str], int]:\n resp = http_con.getresponse()\n resp_body = resp.read()\n resp_headers = {k.lower(): v for k, v in resp.getheaders()}\n return resp_body, resp_headers, resp.status\n\n @classmethod\n def http_con_request(\n cls,\n http_con: http.client.HTTPConnection,\n params: Optional[dict[str, str]] = None,\n *,\n prefix: Optional[str] = None,\n headers: Optional[dict[str, str]] = None,\n method: str = \"GET\",\n body: bytes = b\"\",\n path: str = \"\",\n ) -> tuple[bytes, dict[str, str], int]:\n cls.http_con_send_request(\n http_con,\n params,\n prefix=prefix,\n headers=headers,\n method=method,\n body=body,\n path=path,\n )\n return cls.http_con_read_response(http_con)\n\n @classmethod\n def http_con_json_request(\n cls,\n http_con: http.client.HTTPConnection,\n params: Optional[dict[str, str]] = None,\n *,\n prefix: Optional[str] = None,\n headers: Optional[dict[str, str]] = None,\n body: Any,\n path: str = \"\",\n ):\n response, headers, status = cls.http_con_request(\n http_con,\n params,\n method=\"POST\",\n body=json.dumps(body).encode(),\n prefix=prefix,\n headers={\n \"Content-Type\": \"application/json\",\n **(headers or {}),\n },\n path=path,\n )\n\n if status == http.HTTPStatus.OK:\n result = json.loads(response)\n else:\n result = None\n\n return result, headers, status\n\n @classmethod\n def http_con_binary_request(\n cls,\n http_con: http.client.HTTPConnection,\n query: str,\n proto_ver=edgedb_defines.CURRENT_PROTOCOL,\n bearer_token: Optional[str] = None,\n user: str = \"edgedb\",\n database: str = \"main\",\n ):\n proto_ver_str = f\"v_{proto_ver[0]}_{proto_ver[1]}\"\n mime_type = f\"application/x.edgedb.{proto_ver_str}.binary\"\n headers = {\"Content-Type\": mime_type, \"X-EdgeDB-User\": user}\n if bearer_token:\n headers[\"Authorization\"] = f\"Bearer {bearer_token}\"\n content, headers, status = cls.http_con_request(\n http_con,\n method=\"POST\",\n path=f\"db/{database}\",\n prefix=\"\",\n body=protocol.Execute(\n annotations=[],\n allowed_capabilities=protocol.Capability.ALL,\n compilation_flags=protocol.CompilationFlag(0),\n implicit_limit=0,\n command_text=query,\n input_language=protocol.InputLanguage.EDGEQL,\n output_format=protocol.OutputFormat.JSON,\n expected_cardinality=protocol.Cardinality.AT_MOST_ONE,\n input_typedesc_id=b\"\\0\" * 16,\n output_typedesc_id=b\"\\0\" * 16,\n state_typedesc_id=b\"\\0\" * 16,\n arguments=b\"\",\n state_data=b\"\",\n ).dump() + protocol.Sync().dump(),\n headers=headers,\n )\n content = memoryview(content)\n uint32_unpack = struct.Struct(\"!L\").unpack\n msgs = []\n while content:\n mtype = content[0]\n (msize,) = uint32_unpack(content[1:5])\n msg = protocol.ServerMessage.parse(mtype, content[5: msize + 1])\n msgs.append(msg)\n content = content[msize + 1:]\n return msgs, headers, status\n\n\n_default_cluster = None\n\n\nasync def init_cluster(\n data_dir=None,\n backend_dsn=None,\n *,\n cleanup_atexit=True,\n init_settings=None,\n security=edgedb_args.ServerSecurityMode.Strict,\n http_endpoint_security=edgedb_args.ServerEndpointSecurityMode.Optional,\n compiler_pool_mode=edgedb_args.CompilerPoolMode.Fixed,\n) -> edgedb_cluster.BaseCluster:\n if data_dir is not None and backend_dsn is not None:\n raise ValueError(\n \"data_dir and backend_dsn cannot be set at the same time\")\n if init_settings is None:\n init_settings = {}\n\n log_level = 's' if not debug.flags.server else 'd'\n\n if backend_dsn:\n cluster = edgedb_cluster.TempClusterWithRemotePg(\n backend_dsn,\n testmode=True,\n log_level=log_level,\n data_dir_prefix='edb-test-',\n security=security,\n http_endpoint_security=http_endpoint_security,\n compiler_pool_mode=compiler_pool_mode,\n )\n destroy = True\n elif data_dir is None:\n cluster = edgedb_cluster.TempCluster(\n testmode=True,\n log_level=log_level,\n data_dir_prefix='edb-test-',\n security=security,\n http_endpoint_security=http_endpoint_security,\n compiler_pool_mode=compiler_pool_mode,\n )\n destroy = True\n else:\n cluster = edgedb_cluster.Cluster(\n testmode=True,\n data_dir=data_dir,\n log_level=log_level,\n security=security,\n http_endpoint_security=http_endpoint_security,\n compiler_pool_mode=compiler_pool_mode,\n )\n destroy = False\n\n pg_cluster = await cluster._get_pg_cluster()\n if await pg_cluster.get_status() == 'not-initialized':\n await cluster.init(server_settings=init_settings)\n\n await cluster.start(port=0)\n await cluster.set_test_config()\n await cluster.set_superuser_password('test')\n\n if cleanup_atexit:\n atexit.register(_shutdown_cluster, cluster, destroy=destroy)\n\n return cluster\n\n\ndef _start_cluster(\n *,\n loop: asyncio.AbstractEventLoop,\n cleanup_atexit=True,\n http_endpoint_security=None,\n):\n global _default_cluster\n\n if _default_cluster is None:\n cluster_addr = os.environ.get('EDGEDB_TEST_CLUSTER_ADDR')\n if cluster_addr:\n conn_spec = json.loads(cluster_addr)\n _default_cluster = edgedb_cluster.RunningCluster(**conn_spec)\n else:\n # This branch is not usually used - `edb test` will call\n # init_cluster() separately and set EDGEDB_TEST_CLUSTER_ADDR\n data_dir = os.environ.get('EDGEDB_TEST_DATA_DIR')\n backend_dsn = os.environ.get('EDGEDB_TEST_BACKEND_DSN')\n _default_cluster = loop.run_until_complete(\n init_cluster(\n data_dir=data_dir,\n backend_dsn=backend_dsn,\n cleanup_atexit=cleanup_atexit,\n http_endpoint_security=http_endpoint_security,\n )\n )\n\n return _default_cluster\n\n\ndef _shutdown_cluster(cluster, *, destroy=True):\n global _default_cluster\n _default_cluster = None\n if cluster is not None:\n cluster.stop()\n if destroy:\n cluster.destroy()\n\n\ndef _fetch_metrics(host: str, port: int, sslctx=None) -> str:\n return _call_system_api(\n host, port, '/metrics', return_json=False, sslctx=sslctx\n )\n\n\ndef _fetch_server_info(host: str, port: int) -> dict[str, Any]:\n return _call_system_api(host, port, '/server-info')\n\n\ndef _call_system_api(\n host: str,\n port: int,\n path: str,\n return_json=True,\n sslctx=None,\n **kwargs,\n):\n if sslctx is None:\n con = http.client.HTTPConnection(host, port, **kwargs)\n else:\n con = CustomSNI_HTTPSConnection(host, port, context=sslctx, **kwargs)\n con.connect()\n try:\n con.request(\n 'GET',\n f'http://{host}:{port}{path}'\n )\n resp = con.getresponse()\n if resp.status != 200:\n err = resp.read().decode()\n raise AssertionError(\n f'{path} returned non 200 HTTP status: {resp.status}\\n\\t{err}'\n )\n rv = resp.read().decode()\n if return_json:\n rv = json.loads(rv)\n return rv\n finally:\n con.close()\n\n\ndef parse_metrics(metrics: str) -> dict[str, float]:\n res = {}\n for line in metrics.splitlines():\n if line.startswith('#') or ' ' not in line:\n continue\n key, _, val = line.partition(' ')\n res[key] = float(val)\n return res\n\n\ndef _extract_background_errors(metrics: str) -> str | None:\n non_zero = []\n\n for label, total in parse_metrics(metrics).items():\n if label.startswith('edgedb_server_background_errors_total'):\n if total:\n non_zero.append(\n f'non-zero {label!r} metric: {total}'\n )\n\n if non_zero:\n return '\\n'.join(non_zero)\n else:\n return None\n\n\nasync def drop_db(conn, dbname):\n await conn.execute(f'DROP BRANCH {dbname}')\n\n\nclass ClusterTestCase(TestCaseWithHttpClient):\n\n BASE_TEST_CLASS = True\n backend_dsn: Optional[str] = None\n\n # Some tests may want to manage transactions manually,\n # or affect non-transactional state, in which case\n # TRANSACTION_ISOLATION must be set to False\n TRANSACTION_ISOLATION = True\n\n # By default, tests from the same testsuite may be ran in parallel in\n # several test worker processes. However, certain cases might exhibit\n # pathological locking behavior, or are parallel-unsafe altogether, in\n # which case PARALLELISM_GRANULARITY must be set to 'database', 'suite',\n # or 'system'. The 'database' granularity signals that no two runners\n # may execute tests on the same database in parallel, although the tests\n # may still run on copies of the test database. The 'suite' granularity\n # means that only one test worker is allowed to execute tests from this\n # suite. Finally, the 'system' granularity means that the test suite\n # is not parallelizable at all and must run sequentially with respect\n # to *all other* suites with 'system' granularity.\n PARALLELISM_GRANULARITY = 'default'\n\n # Turns on \"Gel developer\" mode which allows using restricted\n # syntax like USING SQL and similar. It allows modifying standard\n # library (e.g. declaring casts).\n INTERNAL_TESTMODE = True\n\n # Turns off query cache recompilation on DDL\n ENABLE_RECOMPILATION = False\n\n # Setup and teardown commands that run per test\n PER_TEST_SETUP: Sequence[str] = ()\n PER_TEST_TEARDOWN: Sequence[str] = ()\n\n @classmethod\n def setUpClass(cls):\n super().setUpClass()\n cls.cluster = _start_cluster(\n loop=cls.loop,\n cleanup_atexit=True,\n http_endpoint_security=(\n edgedb_args.ServerEndpointSecurityMode.Optional),\n )\n cls.has_create_database = cls.cluster.has_create_database()\n cls.has_create_role = cls.cluster.has_create_role()\n cls.is_superuser = cls.has_create_database and cls.has_create_role\n cls.backend_dsn = os.environ.get('EDGEDB_TEST_BACKEND_DSN')\n if getattr(cls, 'BACKEND_SUPERUSER', False):\n if not cls.is_superuser:\n raise unittest.SkipTest('skipped due to lack of superuser')\n\n @classmethod\n async def tearDownSingleDB(cls):\n await cls.con.execute(\"RESET SCHEMA TO initial;\")\n\n @classmethod\n def fetch_metrics(cls) -> str:\n assert cls.cluster is not None\n conargs = cls.cluster.get_connect_args()\n host, port = conargs['host'], conargs['port']\n ctx = ssl.create_default_context()\n ctx.load_verify_locations(conargs['tls_ca_file'])\n return _fetch_metrics(host, port, sslctx=ctx)\n\n @classmethod\n def get_connect_args(\n cls,\n *,\n cluster=None,\n database=None,\n user=None,\n password=None,\n secret_key=None,\n ):\n if password is None and secret_key is None:\n password = \"test\"\n if cluster is None:\n cluster = cls.cluster\n if database is None:\n database = edgedb_defines.EDGEDB_SUPERUSER_DB\n if user is None:\n user = edgedb_defines.EDGEDB_SUPERUSER\n conargs = cluster.get_connect_args().copy()\n conargs.update(dict(user=user,\n password=password,\n database=database,\n secret_key=secret_key))\n return conargs\n\n @classmethod\n def make_auth_header(cls, user=None, password=None):\n # urllib *does* have actual support for basic auth but it is so much\n # more annoying than just doing it yourself...\n conargs = cls.get_connect_args(user=user, password=password)\n username = conargs.get('user')\n password = conargs.get('password')\n key = f'{username}:{password}'.encode('ascii')\n basic_header = f'Basic {base64.b64encode(key).decode(\"ascii\")}'\n\n return basic_header\n\n @classmethod\n def get_parallelism_granularity(cls):\n if cls.PARALLELISM_GRANULARITY == 'default':\n if cls.TRANSACTION_ISOLATION:\n return 'default'\n else:\n return 'database'\n else:\n return cls.PARALLELISM_GRANULARITY\n\n @classmethod\n def uses_database_copies(cls):\n return (\n os.environ.get('EDGEDB_TEST_PARALLEL')\n and cls.get_parallelism_granularity() == 'database'\n )\n\n def ensure_no_background_server_errors(self):\n metrics = self.fetch_metrics()\n errors = _extract_background_errors(metrics)\n if errors:\n raise AssertionError(\n f'{self._testMethodName!r}:\\n\\n{errors}'\n )\n\n @contextlib.asynccontextmanager\n async def assertRaisesRegexTx(self, exception, regex, msg=None, **kwargs):\n \"\"\"A version of assertRaisesRegex with automatic transaction recovery\n \"\"\"\n\n with super().assertRaisesRegex(exception, regex, msg=msg, **kwargs):\n try:\n tx = self.con.transaction()\n await tx.start()\n yield\n finally:\n await tx.rollback()\n\n @classmethod\n @contextlib.contextmanager\n def http_con(\n cls,\n server=None,\n keep_alive=True,\n client_cert_file=None,\n client_key_file=None,\n **kwargs,\n ):\n if server is None:\n server = cls\n with super().http_con(\n server,\n keep_alive=keep_alive,\n client_cert_file=client_cert_file,\n client_key_file=client_key_file,\n **kwargs,\n ) as http_con:\n yield http_con\n\n @property\n def http_addr(self) -> str:\n conn_args = self.get_connect_args()\n url = f'https://{conn_args[\"host\"]}:{conn_args[\"port\"]}'\n prefix = self.get_api_prefix()\n if prefix:\n url = f'{url}{prefix}'\n return url\n\n @property\n def tls_context(self) -> ssl.SSLContext:\n conn_args = self.get_connect_args()\n tls_context = ssl.create_default_context(\n ssl.Purpose.SERVER_AUTH,\n cafile=conn_args[\"tls_ca_file\"],\n )\n tls_context.check_hostname = False\n return tls_context\n\n\ndef ignore_warnings(warning_message=None):\n def w(f):\n async def wf(self, *args, **kwargs):\n with self.ignore_warnings(warning_message):\n return await f(self, *args, **kwargs)\n\n return wf\n\n return w\n\n\nclass ConnectedTestCase(ClusterTestCase):\n\n BASE_TEST_CLASS = True\n NO_FACTOR = True\n WARN_FACTOR = False\n\n con: tconn.Connection\n\n @classmethod\n def setUpClass(cls):\n super().setUpClass()\n cls.loop.run_until_complete(cls.setup_and_connect())\n\n @classmethod\n def tearDownClass(cls):\n try:\n cls.loop.run_until_complete(cls.teardown_and_disconnect())\n finally:\n super().tearDownClass()\n\n @contextlib.contextmanager\n def ignore_warnings(self, warning_message=None):\n with self.con.capture_warnings() as warnings:\n yield\n\n if warning_message is not None:\n for warning in warnings:\n # If it doesn't match the re, send it back to the con.\n # It might get raised or it might get captured by an\n # enclosing call to capture_warnings/ignore_warnings.\n if not re.search(warning_message, str(warning)):\n self.con._get_warning_handler()([warning], None)\n\n @classmethod\n async def setup_and_connect(cls):\n cls.con = await cls.connect()\n\n @classmethod\n async def teardown_and_disconnect(cls):\n await cls.con.aclose()\n # Give event loop another iteration so that connection\n # transport has a chance to properly close.\n await asyncio.sleep(0)\n cls.con = None\n\n def setUp(self):\n if self.INTERNAL_TESTMODE:\n self.loop.run_until_complete(\n self.con.execute(\n 'CONFIGURE SESSION SET __internal_testmode := true;'))\n\n if not self.ENABLE_RECOMPILATION:\n self.loop.run_until_complete(\n self.con.execute(\n 'CONFIGURE SESSION SET auto_rebuild_query_cache := false;'\n )\n )\n\n if self.NO_FACTOR:\n self.loop.run_until_complete(\n self.con.execute(\n 'CONFIGURE SESSION SET simple_scoping := true;'))\n\n if self.WARN_FACTOR:\n self.loop.run_until_complete(\n self.con.execute(\n 'CONFIGURE SESSION SET warn_old_scoping := true;'))\n\n if self.TRANSACTION_ISOLATION:\n self.xact = self.con.transaction()\n self.loop.run_until_complete(self.xact.start())\n\n for cmd in self.PER_TEST_SETUP:\n self.loop.run_until_complete(self.con.execute(cmd))\n\n super().setUp()\n\n def tearDown(self):\n try:\n self.ensure_no_background_server_errors()\n\n for cmd in self.PER_TEST_TEARDOWN:\n self.loop.run_until_complete(self.con.execute(cmd))\n finally:\n try:\n if self.TRANSACTION_ISOLATION:\n self.loop.run_until_complete(self.xact.rollback())\n del self.xact\n\n if self.con.is_in_transaction():\n self.loop.run_until_complete(\n self.con.query('ROLLBACK'))\n raise AssertionError(\n 'test connection is still in transaction '\n '*after* the test')\n\n if not self.TRANSACTION_ISOLATION:\n self.loop.run_until_complete(\n self.con.execute('RESET ALIAS *;'))\n\n finally:\n super().tearDown()\n\n @classmethod\n async def connect(\n cls,\n *,\n cluster=None,\n database=None,\n user=None,\n password=None,\n secret_key=None,\n ) -> tconn.Connection:\n conargs = cls.get_connect_args(\n cluster=cluster,\n database=database,\n user=user,\n password=password,\n secret_key=secret_key,\n )\n return await tconn.async_connect_test_client(**conargs)\n\n def repl(self):\n \"\"\"Open interactive EdgeQL REPL right in the test.\n\n This is obviously only for debugging purposes. Just add\n `self.repl()` at any point in your test.\n \"\"\"\n\n conargs = self.get_connect_args()\n\n cmd = [\n 'python', '-m', 'edb.cli',\n '--database', self.con.dbname,\n '--user', conargs['user'],\n '--tls-ca-file', conargs['tls_ca_file'],\n ]\n\n env = os.environ.copy()\n env['EDGEDB_HOST'] = conargs['host']\n env['EDGEDB_PORT'] = str(conargs['port'])\n if password := conargs.get('password'):\n env['EDGEDB_PASSWORD'] = password\n if secret_key := conargs.get('secret_key'):\n env['EDGEDB_SECRET_KEY'] = secret_key\n\n proc = subprocess.Popen(\n cmd, stdin=sys.stdin, stdout=sys.stdout, env=env)\n while proc.returncode is None:\n try:\n proc.wait()\n except KeyboardInterrupt:\n pass\n\n def _run_and_rollback(self):\n return RollbackChanges(self.con)\n\n async def _run_and_rollback_retrying(self):\n @contextlib.asynccontextmanager\n async def cm(tx):\n try:\n async with tx:\n await tx._ensure_transaction()\n yield tx\n raise RollbackException\n except RollbackException:\n pass\n\n async for tx in self.con.retrying_transaction():\n yield cm(tx)\n\n def assert_data_shape(self, data, shape,\n message=None, rel_tol=None, abs_tol=None):\n assert_data_shape.assert_data_shape(\n data, shape, self.fail,\n message=message, rel_tol=rel_tol, abs_tol=abs_tol,\n )\n\n async def assert_query_result(\n self,\n query,\n exp_result_json,\n exp_result_binary=...,\n *,\n always_typenames=False,\n always_typeids=False,\n msg=None,\n sort=None,\n implicit_limit=0,\n variables=None,\n json_only=False,\n binary_only=False,\n rel_tol=None,\n abs_tol=None,\n language: Literal[\"sql\", \"edgeql\"] = \"edgeql\",\n ):\n fetch_args = variables if isinstance(variables, tuple) else ()\n fetch_kw = variables if isinstance(variables, dict) else {}\n\n if not binary_only and language != \"sql\":\n try:\n tx = self.con.transaction()\n await tx.start()\n try:\n res = await self.con._fetchall_json(\n query,\n *fetch_args,\n __limit__=implicit_limit,\n **fetch_kw)\n finally:\n await tx.rollback()\n\n res = json.loads(res)\n if sort is not None:\n assert_data_shape.sort_results(res, sort)\n assert_data_shape.assert_data_shape(\n res, exp_result_json, self.fail,\n message=msg, rel_tol=rel_tol, abs_tol=abs_tol,\n )\n except Exception:\n self.add_fail_notes(serialization='json')\n if msg:\n self.add_fail_notes(msg=msg)\n raise\n\n if json_only:\n return\n\n if exp_result_binary is ...:\n # The expected result is the same\n exp_result_binary = exp_result_json\n\n typenames = random.choice([True, False]) or always_typenames\n typeids = random.choice([True, False]) or always_typeids\n\n try:\n res = await self.con._fetchall(\n query,\n *fetch_args,\n __typenames__=typenames,\n __typeids__=typeids,\n __limit__=implicit_limit,\n __language__=(\n tconn.InputLanguage.SQL if language == \"sql\"\n else tconn.InputLanguage.EDGEQL\n ),\n **fetch_kw\n )\n res = serutils.serialize(res)\n if sort is not None:\n assert_data_shape.sort_results(res, sort)\n assert_data_shape.assert_data_shape(\n res,\n exp_result_binary,\n self.fail,\n message=msg,\n rel_tol=rel_tol,\n abs_tol=abs_tol,\n )\n except Exception:\n self.add_fail_notes(\n serialization='binary',\n __typenames__=typenames,\n __typeids__=typeids)\n if msg:\n self.add_fail_notes(msg=msg)\n raise\n\n async def assert_sql_query_result(\n self,\n query,\n exp_result,\n *,\n implicit_limit=0,\n msg=None,\n sort=None,\n variables=None,\n rel_tol=None,\n abs_tol=None,\n apply_access_policies=True,\n ):\n if not apply_access_policies:\n ctx = self.without_access_policies()\n else:\n ctx = contextlib.nullcontext()\n async with ctx:\n await self.assert_query_result(\n query,\n exp_result,\n implicit_limit=implicit_limit,\n msg=msg,\n sort=sort,\n variables=variables,\n rel_tol=rel_tol,\n abs_tol=abs_tol,\n language=\"sql\",\n )\n\n async def assert_index_use(self, query, *args, index_type):\n def look(obj):\n if (\n isinstance(obj, dict)\n and \"IndexScan\" in obj.get('plan_type', '')\n ):\n return any(\n prop['title'] == 'index_name'\n and index_type in prop['value']\n for prop in obj.get('properties', [])\n )\n\n if isinstance(obj, dict):\n return any([look(v) for v in obj.values()])\n elif isinstance(obj, list):\n return any(look(v) for v in obj)\n else:\n return False\n\n plan = await self.con.query_json(f'analyze {query}', *args)\n if not look(json.loads(plan)):\n raise AssertionError(f\"query did not use the {index_type!r} index\")\n\n @classmethod\n def get_backend_sql_dsn(cls, dbname=None):\n settings = cls.con.get_settings()\n pgdsn = settings.get('pgdsn')\n if pgdsn is None:\n raise unittest.SkipTest('raw SQL test skipped: not in devmode')\n params = ConnectionParams(dsn=pgdsn.decode('utf8'))\n if dbname:\n params.update(database=dbname)\n params.clear_server_settings()\n return params.to_dsn()\n\n @classmethod\n async def get_backend_sql_connection(cls, dbname=None):\n \"\"\"Get a raw connection to the underlying SQL server, if possible\n\n This is useful when we want to do things like querying the pg_catalog\n of the underlying database.\n \"\"\"\n try:\n import asyncpg\n except ImportError:\n raise unittest.SkipTest(\n 'SQL test skipped: asyncpg not installed')\n\n pgdsn = cls.get_backend_sql_dsn(dbname=dbname)\n return await asyncpg.connect(pgdsn)\n\n @classmethod\n @contextlib.asynccontextmanager\n async def with_backend_sql_connection(cls, dbname=None):\n con = await cls.get_backend_sql_connection(dbname=dbname)\n try:\n yield con\n finally:\n await con.close()\n\n @contextlib.asynccontextmanager\n async def without_access_policies(self):\n await self.con.execute(\n 'CONFIGURE SESSION SET apply_access_policies := false'\n )\n raised_an_execption = False\n try:\n yield\n except BaseException:\n raised_an_execption = True\n raise\n finally:\n if not (raised_an_execption and self.con.is_in_transaction()):\n await self.con.execute(\n 'CONFIGURE SESSION RESET apply_access_policies'\n )\n\n @classmethod\n def get_sql_proto_dsn(cls, dbname=None):\n dbname = dbname or cls.con.dbname\n conargs = cls.get_connect_args()\n return (\n f\"postgres://{conargs['user']}:{conargs['password']}@\"\n f\"{conargs['host']}:{conargs['port']}/{cls.con.dbname}?\"\n f\"sslrootcert={conargs['tls_ca_file']}\"\n )\n\n\nclass DatabaseTestCase(ConnectedTestCase):\n\n SETUP: Optional[str | pathlib.Path | list[str] | list[pathlib.Path]] = None\n TEARDOWN: Optional[str] = None\n SCHEMA: Optional[str | pathlib.Path] = None\n DEFAULT_MODULE: str = 'default'\n EXTENSIONS: list[str] = []\n\n BASE_TEST_CLASS = True\n\n con: Any # XXX: the real type?\n\n @classmethod\n async def setup_and_connect(cls):\n dbname = cls.get_database_name()\n\n cls.con = None\n\n class_set_up = os.environ.get('EDGEDB_TEST_CASES_SET_UP', 'run')\n\n # Only open an extra admin connection if necessary.\n if class_set_up == 'run':\n script = f'CREATE DATABASE {dbname};'\n admin_conn = await cls.connect(\n database=edgedb_defines.EDGEDB_SUPERUSER_DB\n )\n await admin_conn.execute(script)\n await admin_conn.aclose()\n\n elif class_set_up == 'inplace':\n dbname = edgedb_defines.EDGEDB_SUPERUSER_DB\n\n elif cls.uses_database_copies():\n admin_conn = await cls.connect(\n database=edgedb_defines.EDGEDB_SUPERUSER_DB\n )\n\n base_db_name, _, _ = dbname.rpartition('_')\n\n if cls.get_setup_script():\n await admin_conn.execute('''\n configure session set __internal_testmode := true;\n ''')\n\n create_command = (\n f'CREATE TEMPLATE BRANCH {qlquote.quote_ident(dbname)}'\n f' FROM {qlquote.quote_ident(base_db_name)};'\n )\n else:\n create_command = (\n f'CREATE EMPTY BRANCH {qlquote.quote_ident(dbname)}')\n\n # The retry here allows the test to survive a concurrent testing\n # Gel server (e.g. async with tb.start_edgedb_server()) whose\n # introspection holds a lock on the base_db here\n async for tr in cls.try_until_succeeds(\n ignore=edgedb.ExecutionError,\n timeout=30,\n ):\n async with tr:\n await admin_conn.execute(create_command)\n\n await admin_conn.aclose()\n\n cls.con = await cls.connect(database=dbname)\n\n if class_set_up != 'skip':\n script = cls.get_setup_script()\n if script:\n with cls.con.ignore_warnings():\n await cls.con.execute(script)\n\n @staticmethod\n def get_set_up():\n return os.environ.get('EDGEDB_TEST_CASES_SET_UP', 'run')\n\n @classmethod\n async def teardown_and_disconnect(cls):\n script = ''\n\n class_set_up = cls.get_set_up()\n\n if cls.TEARDOWN and class_set_up != 'skip':\n script = cls.TEARDOWN.strip()\n\n try:\n if script:\n await cls.con.execute(script)\n if class_set_up == 'inplace':\n await cls.tearDownSingleDB()\n finally:\n await cls.con.aclose()\n\n if class_set_up == 'inplace':\n pass\n\n elif class_set_up == 'run' or cls.uses_database_copies():\n dbname = qlquote.quote_ident(cls.get_database_name())\n admin_conn = await cls.connect(\n database=edgedb_defines.EDGEDB_SUPERUSER_DB\n )\n try:\n await drop_db(admin_conn, dbname)\n finally:\n await admin_conn.aclose()\n\n @classmethod\n def get_connect_args(\n cls,\n *,\n database=None,\n **kwargs,\n ):\n return super().get_connect_args(\n database=database or cls.get_database_name(),\n **kwargs,\n )\n\n @classmethod\n def get_database_name(cls):\n if not getattr(cls, 'has_create_database', True):\n return edgedb_defines.EDGEDB_SUPERUSER_DB\n\n if cls.__name__.startswith('TestEdgeQL'):\n dbname = cls.__name__[len('TestEdgeQL'):]\n elif cls.__name__.startswith('Test'):\n dbname = cls.__name__[len('Test'):]\n else:\n dbname = cls.__name__\n\n if cls.uses_database_copies():\n return f'{dbname.lower()}_{os.getpid()}'\n else:\n return dbname.lower()\n\n @classmethod\n def get_api_prefix(cls):\n return f'/db/{cls.get_database_name()}'\n\n @classmethod\n def get_setup_script(cls):\n script = ''\n has_nontrivial_script = False\n\n # allow the setup script to also run in test mode and no recompilation\n if cls.INTERNAL_TESTMODE:\n script += '\\nCONFIGURE SESSION SET __internal_testmode := true;'\n if not cls.ENABLE_RECOMPILATION:\n script += (\n '\\nCONFIGURE SESSION SET auto_rebuild_query_cache := false;'\n )\n\n if getattr(cls, 'BACKEND_SUPERUSER', False):\n is_superuser = getattr(cls, 'is_superuser', True)\n if not is_superuser:\n raise unittest.SkipTest('skipped due to lack of superuser')\n\n schema = []\n # Incude the extensions before adding schemas.\n for ext in cls.EXTENSIONS:\n schema.append(f'using extension {ext};')\n\n # Look at all SCHEMA entries and potentially create multiple\n # modules, but always create the test module, if not `default`.\n if cls.DEFAULT_MODULE != 'default':\n schema.append(f'\\nmodule {cls.DEFAULT_MODULE} {{}}')\n for name in dir(cls):\n m = re.match(r'^SCHEMA(?:_(\\w+))?', name)\n if m:\n module_name = (\n (m.group(1) or cls.DEFAULT_MODULE)\n .lower().replace('_', '::')\n )\n\n schema_fn = getattr(cls, name)\n if schema_fn is not None:\n with open(schema_fn, 'r') as sf:\n module = sf.read()\n\n schema.append(f'\\nmodule {module_name} {{ {module} }}')\n\n full_schema_fn = getattr(cls, 'FULL_SCHEMA', None)\n if full_schema_fn:\n with open(full_schema_fn, 'r') as sf:\n schema.append(sf.read())\n\n if schema:\n has_nontrivial_script = True\n\n script += f'\\nSTART MIGRATION'\n script += f' TO {{ {\"\".join(schema)} }};'\n script += f'\\nPOPULATE MIGRATION;'\n script += f'\\nCOMMIT MIGRATION;'\n\n if cls.SETUP:\n if not isinstance(cls.SETUP, (list, tuple)):\n scripts = [cls.SETUP]\n else:\n scripts = cls.SETUP\n\n for scr in scripts:\n has_nontrivial_script = True\n\n is_path = (\n isinstance(scr, pathlib.Path)\n or '\\n' not in scr and os.path.exists(scr)\n )\n\n if is_path:\n with open(scr, 'rt') as f:\n setup_text = f.read()\n else:\n assert isinstance(scr, str)\n setup_text = scr\n\n script += '\\n' + setup_text\n\n # If the SETUP script did a SET MODULE, make sure it is cleared\n # (since in some modes we keep using the same connection)\n script += '\\nRESET MODULE;'\n\n # allow the setup script to also run in test mode\n if cls.INTERNAL_TESTMODE:\n script += '\\nCONFIGURE SESSION SET __internal_testmode := false;'\n if not cls.ENABLE_RECOMPILATION:\n script += '\\nCONFIGURE SESSION RESET auto_rebuild_query_cache;'\n\n return script.strip(' \\n') if has_nontrivial_script else ''\n\n async def migrate(self, migration, *, module: Optional[str] = 'default'):\n if module:\n migration = f\"\"\"\n module {module} {{\n {migration}\n }}\n \"\"\"\n\n with self.ignore_warnings('Non-simple_scoping will be removed'):\n await self.con.execute(f\"\"\"\n START MIGRATION TO {{\n {migration}\n }};\n POPULATE MIGRATION;\n COMMIT MIGRATION;\n \"\"\")\n\n\nclass Error:\n def __init__(self, cls, message, shape):\n self._message = message\n self._class = cls\n self._shape = shape\n\n @property\n def message(self):\n return self._message\n\n @property\n def cls(self):\n return self._class\n\n @property\n def shape(self):\n return self._shape\n\n\nclass BaseQueryTestCase(DatabaseTestCase):\n\n BASE_TEST_CLASS = True\n\n\nclass DDLTestCase(BaseQueryTestCase):\n # DDL test cases generally need to be serialized\n # to avoid deadlocks in parallel execution.\n PARALLELISM_GRANULARITY = 'database'\n BASE_TEST_CLASS = True\n\n\nclass QueryTestCase(BaseQueryTestCase):\n\n BASE_TEST_CLASS = True\n\n\nclass SQLQueryTestCase(BaseQueryTestCase):\n\n BASE_TEST_CLASS = True\n\n scon: asyncpg.Connection\n\n @classmethod\n def setUpClass(cls):\n try:\n import asyncpg # noqa: F401\n except ImportError:\n raise unittest.SkipTest('SQL tests skipped: asyncpg not installed')\n\n super().setUpClass()\n cls.scon = cls.loop.run_until_complete(\n cls.create_sql_connection()\n )\n\n @classmethod\n def create_sql_connection(\n cls,\n *,\n user: str = None,\n password: str = None,\n ) -> asyncio.Future[asyncpg.Connection]:\n import asyncpg\n conargs = cls.get_connect_args()\n\n tls_context = ssl.create_default_context(\n ssl.Purpose.SERVER_AUTH,\n cafile=conargs[\"tls_ca_file\"],\n )\n tls_context.check_hostname = False\n\n return asyncpg.connect(\n host=conargs['host'],\n port=conargs['port'],\n user=conargs['user'] if user is None else user,\n password=conargs['password'] if password is None else password,\n database=cls.con.dbname,\n ssl=tls_context,\n )\n\n @classmethod\n def tearDownClass(cls):\n try:\n cls.loop.run_until_complete(cls.scon.close())\n # Give event loop another iteration so that connection\n # transport has a chance to properly close.\n cls.loop.run_until_complete(asyncio.sleep(0))\n cls.scon = None\n finally:\n super().tearDownClass()\n\n def setUp(self):\n if self.TRANSACTION_ISOLATION:\n self.stran = self.scon.transaction()\n self.loop.run_until_complete(self.stran.start())\n super().setUp()\n\n def tearDown(self):\n try:\n if self.TRANSACTION_ISOLATION:\n self.loop.run_until_complete(self.stran.rollback())\n self.loop.run_until_complete(self.scon.execute('RESET ALL'))\n finally:\n super().tearDown()\n\n async def squery_values(self, query, *args):\n res = await self.scon.fetch(query, *args)\n return [list(r.values()) for r in res]\n\n def assert_shape(self, res: Any, rows: int, columns: int | list[str]):\n \"\"\"\n Fail if query result does not confront the specified shape, defined in\n terms of:\n - number of rows,\n - number of columns (not checked if there are not rows)\n - column names.\n \"\"\"\n\n self.assertEqual(len(res), rows)\n\n if isinstance(columns, int):\n if rows > 0:\n self.assertEqual(len(res[0]), columns)\n elif isinstance(columns, list):\n self.assertListEqual(columns, list(res[0].keys()))\n\n\nclass CLITestCaseMixin:\n\n def run_cli(self, *args, input: Optional[str] = None) -> None:\n conn_args = self.get_connect_args()\n self.run_cli_on_connection(conn_args, *args, input=input)\n\n @classmethod\n def run_cli_on_connection(\n cls, conn_args: dict[str, Any], *args, input: Optional[str] = None\n ) -> None:\n cmd_args = [\n '--host', conn_args['host'],\n '--port', str(conn_args['port']),\n '--tls-ca-file', conn_args['tls_ca_file']\n ]\n if conn_args.get('user'):\n cmd_args += ['--user', conn_args['user']]\n if conn_args.get('password'):\n cmd_args += ['--password-from-stdin']\n if input is not None:\n input = f\"{conn_args['password']}\\n{input}\"\n else:\n input = f\"{conn_args['password']}\\n\"\n cmd_args += args\n cmd = ['gel'] + cmd_args\n try:\n subprocess.run(\n cmd,\n input=input.encode() if input else None,\n check=True,\n capture_output=True,\n )\n except subprocess.CalledProcessError as e:\n output = '\\n'.join(getattr(out, 'decode', out.__str__)()\n for out in [e.output, e.stderr] if out)\n raise AssertionError(\n f'command {cmd} returned non-zero exit status {e.returncode}'\n f'\\n{output}'\n ) from e\n\n\nclass DumpCompatTestCaseMeta(TestCaseMeta):\n\n def __new__(\n mcls,\n name,\n bases,\n ns,\n *,\n dump_subdir=None,\n check_method=None,\n ):\n if not name.startswith('Test'):\n return super().__new__(mcls, name, bases, ns)\n\n if dump_subdir is None:\n raise TypeError(\n f'{name}: missing required \"dump_subdir\" class argument')\n\n if check_method is None:\n raise TypeError(\n f'{name}: missing required \"check_method\" class argument')\n\n mod = sys.modules[ns['__module__']]\n dumps_dir = pathlib.Path(mod.__file__).parent / 'dumps' / dump_subdir\n\n async def check_dump_restore_compat_single_db(self, *, dumpfn):\n dbname = edgedb_defines.EDGEDB_SUPERUSER_DB\n self.run_cli('-d', dbname, 'restore', str(dumpfn))\n try:\n await check_method(self)\n finally:\n await self.tearDownSingleDB()\n\n async def check_dump_restore_compat(self, *, dumpfn: pathlib.Path):\n if not self.has_create_database:\n return await check_dump_restore_compat_single_db(\n self, dumpfn=dumpfn\n )\n\n dbname = f\"{type(self).__name__}_{dumpfn.stem}\"\n qdbname = qlquote.quote_ident(dbname)\n await self.con.execute(f'CREATE DATABASE {qdbname}')\n try:\n self.run_cli('-d', dbname, 'restore', str(dumpfn))\n con2 = await self.connect(database=dbname)\n except Exception:\n await drop_db(self.con, qdbname)\n raise\n\n oldcon = self.__class__.con\n self.__class__.con = con2\n try:\n await check_method(self)\n finally:\n self.__class__.con = oldcon\n await con2.aclose()\n\n await drop_db(self.con, qdbname)\n\n for entry in dumps_dir.iterdir():\n if not entry.is_file() or not entry.name.endswith(\".dump\"):\n continue\n\n mcls.add_method(\n f'test_{dump_subdir}_restore_compatibility_{entry.stem}',\n ns,\n functools.partial(check_dump_restore_compat, dumpfn=entry),\n )\n\n return super().__new__(mcls, name, bases, ns)\n\n\nclass DumpCompatTestCase(\n ConnectedTestCase,\n CLITestCaseMixin,\n metaclass=DumpCompatTestCaseMeta,\n):\n BASE_TEST_CLASS = True\n TRANSACTION_ISOLATION = False\n\n\nclass StableDumpTestCase(QueryTestCase, CLITestCaseMixin):\n\n BASE_TEST_CLASS = True\n STABLE_DUMP = True\n TRANSACTION_ISOLATION = False\n PARALLELISM_GRANULARITY = 'suite'\n\n async def check_dump_restore_single_db(self, check_method):\n with tempfile.TemporaryDirectory() as f:\n fname = os.path.join(f, 'dump')\n dbname = edgedb_defines.EDGEDB_SUPERUSER_DB\n await asyncio.to_thread(self.run_cli, '-d', dbname, 'dump', fname)\n await self.tearDownSingleDB()\n await asyncio.to_thread(\n self.run_cli, '-d', dbname, 'restore', fname\n )\n\n # Cycle the connection to avoid state mismatches\n await self.con.aclose()\n self.con = await self.connect(database=dbname)\n\n await check_method(self)\n\n async def check_dump_restore(\n self, check_method, include_secrets: bool=False\n ):\n if not self.has_create_database:\n return await self.check_dump_restore_single_db(check_method)\n\n src_dbname = self.get_database_name()\n tgt_dbname = f'{src_dbname}_restored'\n q_tgt_dbname = qlquote.quote_ident(tgt_dbname)\n with tempfile.TemporaryDirectory() as f:\n fname = os.path.join(f, 'dump')\n extra = ['--include-secrets'] if include_secrets else []\n await asyncio.to_thread(\n self.run_cli, '-d', src_dbname, 'dump', fname, *extra\n )\n\n await self.con.execute(f'CREATE DATABASE {q_tgt_dbname}')\n try:\n await asyncio.to_thread(\n self.run_cli, '-d', tgt_dbname, 'restore', fname\n )\n con2 = await self.connect(database=tgt_dbname)\n except Exception:\n await drop_db(self.con, q_tgt_dbname)\n raise\n\n oldcon = self.con\n self.__class__.con = con2\n try:\n await check_method(self)\n finally:\n self.__class__.con = oldcon\n await con2.aclose()\n await drop_db(self.con, q_tgt_dbname)\n\n async def check_branching(self, include_data=False, *, check_method):\n if not self.has_create_database:\n self.skipTest(\"create branch is not supported by the backend\")\n\n orig_branch = self.get_database_name()\n new_branch = f'new_{orig_branch}'\n # record the original schema\n orig_schema = await self.con.query_single('describe schema as sdl')\n\n # connect to a default branch so we can create a new branch\n branch_type = 'data' if include_data else 'schema'\n await self.con.execute(\n f'create {branch_type} branch {new_branch} '\n f'from {orig_branch}'\n )\n\n try:\n con2 = await self.connect(database=new_branch)\n except Exception:\n await drop_db(self.con, new_branch)\n raise\n\n oldcon = self.con\n self.__class__.con = con2\n try:\n # We cannot compare the SDL text of the new branch schema to the\n # original because the order in which it renders all the\n # components is not guaranteed. Instead we will use migrations to\n # compare the new branch schema to the original. We expect there\n # to be no difference and therefore a new migration to the\n # original schema should have the \"complete\" status right away.\n with self.ignore_warnings():\n await self.con.execute(\n f'start migration to {{ {orig_schema} }}'\n )\n mig_status = json.loads(\n await self.con.query_single_json(\n 'describe current migration as json'\n )\n )\n self.assertTrue(mig_status.get('complete'))\n await self.con.execute('abort migration')\n\n # run the check_method on the copied branch\n if include_data:\n await check_method(self)\n else:\n await check_method(self, include_data=include_data)\n finally:\n self.__class__.con = oldcon\n await con2.aclose()\n await drop_db(self.con, new_branch)\n\n\nclass StablePGDumpTestCase(BaseQueryTestCase):\n\n BASE_TEST_CLASS = True\n TRANSACTION_ISOLATION = False\n\n def run_pg_dump(self, *args, input: Optional[str] = None) -> None:\n conargs = self.get_connect_args()\n self.run_pg_dump_on_connection(conargs, *args, input=input)\n\n @classmethod\n def run_pg_dump_on_connection(\n cls, dsn: str, *args, input: Optional[str] = None\n ) -> None:\n cmd = [cls._pg_bin_dir / 'pg_dump', '--dbname', dsn]\n cmd += args\n try:\n subprocess.run(\n cmd,\n input=input.encode() if input else None,\n check=True,\n capture_output=True,\n )\n except subprocess.CalledProcessError as e:\n output = '\\n'.join(getattr(out, 'decode', out.__str__)()\n for out in [e.output, e.stderr] if out)\n raise AssertionError(\n f'command {cmd} returned non-zero exit status {e.returncode}'\n f'\\n{output}'\n ) from e\n\n @classmethod\n def setUpClass(cls):\n try:\n import asyncpg\n except ImportError:\n raise unittest.SkipTest('SQL tests skipped: asyncpg not installed')\n\n if cls.get_set_up() == 'inplace':\n raise unittest.SkipTest('SQL dump tests skipped in single db mode')\n\n super().setUpClass()\n frontend_dsn = cls.get_sql_proto_dsn()\n src_dbname = cls.con.dbname\n tgt_dbname = f'restored_{src_dbname}'\n try:\n newdsn = cls.get_backend_sql_dsn(dbname=tgt_dbname)\n except Exception:\n super().tearDownClass()\n raise\n\n cls._pg_bin_dir = cls.loop.run_until_complete(\n pgcluster.get_pg_bin_dir())\n\n cls.backend = cls.loop.run_until_complete(\n cls.get_backend_sql_connection())\n\n # Run pg_dump to create the dump data for an existing Gel database.\n with tempfile.NamedTemporaryFile() as f:\n cls.run_pg_dump_on_connection(frontend_dsn, '-f', f.name)\n\n # Skip the restore part of the test if the database\n # backend is older than our pg_dump, since it won't work.\n pg_ver_str = cls.loop.run_until_complete(\n cls.backend.fetch('select version()')\n )[0][0]\n pg_ver = buildmeta.parse_pg_version(pg_ver_str)\n bundled_ver = buildmeta.get_pg_version()\n if pg_ver.major < bundled_ver.major:\n raise unittest.SkipTest('pg_dump newer than backend')\n\n # Create a new Postgres database to be used for dump tests.\n db_exists = cls.loop.run_until_complete(\n cls.backend.fetch(f'''\n SELECT oid\n FROM pg_database\n WHERE datname = {tgt_dbname!r}\n ''')\n )\n if list(db_exists):\n cls.loop.run_until_complete(\n cls.backend.execute(f'drop database {tgt_dbname}')\n )\n cls.loop.run_until_complete(\n cls.backend.execute(f'create database {tgt_dbname}')\n )\n\n # Populate the new database using the dump\n cmd = [\n cls._pg_bin_dir / 'psql',\n '-a',\n '--dbname', newdsn,\n '-f', f.name,\n '-v', 'ON_ERROR_STOP=on',\n ]\n try:\n subprocess.run(\n cmd,\n input=None,\n check=True,\n capture_output=True,\n )\n except subprocess.CalledProcessError as e:\n output = '\\n'.join(getattr(out, 'decode', out.__str__)()\n for out in [e.output, e.stderr] if out)\n raise AssertionError(\n f'command {cmd} returned non-zero exit status '\n f'{e.returncode}\\n{output}'\n ) from e\n\n # Connect to the newly created database.\n cls.scon = cls.loop.run_until_complete(\n asyncpg.connect(newdsn))\n\n @classmethod\n def tearDownClass(cls):\n try:\n cls.loop.run_until_complete(cls.scon.close())\n # Give event loop another iteration so that connection\n # transport has a chance to properly close.\n cls.loop.run_until_complete(asyncio.sleep(0))\n cls.scon = None\n\n tgt_dbname = f'restored_{cls.con.dbname}'\n cls.loop.run_until_complete(\n cls.backend.execute(f'drop database {tgt_dbname}')\n )\n cls.loop.run_until_complete(cls.backend.close())\n cls.loop.run_until_complete(asyncio.sleep(0))\n finally:\n super().tearDownClass()\n\n def assert_shape(\n self,\n sqlres: Iterable[Any],\n eqlres: Iterable[asyncpg.Record],\n ) -> None:\n \"\"\"\n Compare the shape of results produced by a SQL query and an EdgeQL\n query.\n \"\"\"\n\n assert_data_shape.assert_data_shape(\n list(sqlres),\n [dataclasses.asdict(r) for r in eqlres],\n self.fail,\n from_sql=True,\n )\n\n def multi_prop_subquery(self, source: str, prop: str) -> str:\n \"Propduce a subquery fetching a multi prop as an array.\"\n\n return (\n f'(SELECT array_agg(target) FROM \"{source}.{prop}\"'\n f' WHERE source = \"{source}\".id) AS {prop}'\n )\n\n def single_link_subquery(\n self,\n source: str,\n link: str,\n target: str,\n link_props: Optional[Iterable[str]] = None\n ) -> str:\n \"\"\"Propduce a subquery fetching a single link as a record.\n\n If no link properties are specified then the array of records will be\n made up of target types.\n\n If the link properties are specified then the array of records will be\n made up of link records.\n \"\"\"\n\n if link_props:\n return (\n f'(SELECT x FROM \"{target}\"'\n f' JOIN \"{source}.{link}\" x ON x.target = \"{target}\".id'\n f' WHERE x.source = \"{source}\".id) AS _{link}'\n )\n\n else:\n return (\n f'(SELECT \"{target}\" FROM \"{target}\"'\n f' WHERE \"{target}\".id = \"{source}\".{link}_id) AS {link}'\n )\n\n def multi_link_subquery(\n self,\n source: str,\n link: str,\n target: str,\n link_props: Optional[Iterable[str]] = None\n ) -> str:\n \"\"\"Propduce a subquery fetching a multi link as an array or records.\n\n If no link properties are specified then the array of records will be\n made up of target types.\n\n If the link properties are specified then the array of records will be\n made up of link records.\n \"\"\"\n\n if link_props:\n return (\n f'(SELECT array_agg(x) FROM \"{target}\"'\n f' JOIN \"{source}.{link}\" x ON x.target = \"{target}\".id'\n f' WHERE x.source = \"{source}\".id) AS _{link}'\n )\n\n else:\n return (\n f'(SELECT array_agg(\"{target}\") FROM \"{target}\"'\n f' JOIN \"{source}.{link}\" x ON x.target = \"{target}\".id'\n f' WHERE x.source = \"{source}\".id) AS {link}'\n )\n\n\ndef get_test_cases_setup(\n cases: Iterable[unittest.TestCase]\n) -> list[tuple[unittest.TestCase, DatabaseName, SetupScript]]:\n result: list[tuple[unittest.TestCase, DatabaseName, SetupScript]] = []\n\n for case in cases:\n if not hasattr(case, 'get_setup_script'):\n continue\n\n try:\n setup_script = case.get_setup_script()\n except unittest.SkipTest:\n continue\n\n dbname = case.get_database_name()\n result.append((case, dbname, setup_script))\n\n return result\n\n\ndef test_cases_use_server(cases: Iterable[unittest.TestCase]) -> bool:\n for case in cases:\n if not hasattr(case, 'uses_server'):\n continue\n\n if case.uses_server():\n return True\n\n\nasync def setup_test_cases(\n cases,\n conn,\n num_jobs,\n try_cached_db=False,\n skip_empty_databases=False,\n verbose=False,\n):\n setup = get_test_cases_setup(cases)\n\n stats = []\n if num_jobs == 1:\n # Special case for --jobs=1\n for _case, dbname, setup_script in setup:\n if skip_empty_databases and not setup_script:\n continue\n await _setup_database(\n dbname, setup_script, conn, stats, try_cached_db)\n if verbose:\n print(f' -> {dbname}: OK', flush=True)\n else:\n async with asyncio.TaskGroup() as g:\n # Use a semaphore to limit the concurrency of bootstrap\n # tasks to the number of jobs (bootstrap is heavy, having\n # more tasks than `--jobs` won't necessarily make\n # things faster.)\n sem = asyncio.BoundedSemaphore(num_jobs)\n\n async def controller(coro, dbname, *args):\n async with sem:\n await coro(dbname, *args)\n if verbose:\n print(f' -> {dbname}: OK', flush=True)\n\n for _case, dbname, setup_script in setup:\n if skip_empty_databases and not setup_script:\n continue\n\n g.create_task(controller(\n _setup_database, dbname, setup_script, conn, stats,\n try_cached_db))\n return stats\n\n\nasync def _setup_database(\n dbname, setup_script, conn_args, stats, try_cached_db):\n start_time = time.monotonic()\n default_args = {\n 'user': edgedb_defines.EDGEDB_SUPERUSER,\n 'password': 'test',\n }\n\n default_args.update(conn_args)\n\n try:\n admin_conn = await tconn.async_connect_test_client(\n database=edgedb_defines.EDGEDB_SUPERUSER_DB,\n **default_args)\n except Exception as ex:\n raise RuntimeError(\n f'exception during creation of {dbname!r} test DB; '\n f'could not connect to the {edgedb_defines.EDGEDB_SUPERUSER_DB} '\n f'db; {type(ex).__name__}({ex})'\n ) from ex\n\n try:\n await admin_conn.execute(\n f'CREATE DATABASE {qlquote.quote_ident(dbname)};'\n )\n except edgedb.DuplicateDatabaseDefinitionError:\n # Eh, that's fine\n # And, if we are trying to use a cache of the database, assume\n # the db is populated and return.\n if try_cached_db:\n elapsed = time.monotonic() - start_time\n stats.append(\n ('setup::' + dbname,\n {'running-time': elapsed, 'cached': True}))\n return\n except Exception as ex:\n raise RuntimeError(\n f'exception during creation of {dbname!r} test DB: '\n f'{type(ex).__name__}({ex})'\n ) from ex\n finally:\n await admin_conn.aclose()\n\n dbconn = await tconn.async_connect_test_client(\n database=dbname, **default_args\n )\n try:\n if setup_script:\n async for tx in dbconn.retrying_transaction():\n async with tx:\n with dbconn.capture_warnings():\n await dbconn.execute(setup_script)\n except Exception as ex:\n raise RuntimeError(\n f'exception during initialization of {dbname!r} test DB: '\n f'{type(ex).__name__}({ex})'\n ) from ex\n finally:\n await dbconn.aclose()\n\n elapsed = time.monotonic() - start_time\n stats.append(\n ('setup::' + dbname, {'running-time': elapsed, 'cached': False}))\n\n\n_lock_cnt = 0\n\n\ndef gen_lock_key():\n global _lock_cnt\n _lock_cnt += 1\n return os.getpid() * 1000 + _lock_cnt\n\n\nclass _EdgeDBServerData(NamedTuple):\n\n host: str\n port: int\n password: str\n server_data: Any\n tls_cert_file: str\n pid: int\n\n def get_connect_args(self, **kwargs) -> dict[str, str | int]:\n conn_args = dict(\n user='edgedb',\n password=self.password,\n host=self.host,\n port=self.port,\n tls_ca_file=self.tls_cert_file,\n )\n\n conn_args.update(kwargs)\n return conn_args\n\n def fetch_metrics(self) -> str:\n ctx = ssl.create_default_context()\n ctx.load_verify_locations(self.tls_cert_file)\n return _fetch_metrics(self.host, self.port, sslctx=ctx)\n\n def fetch_server_info(self) -> dict[str, Any]:\n return _fetch_server_info(self.host, self.port)\n\n def call_system_api(self, path: str, **kwargs):\n args = dict(host=self.host, port=self.port, path=path)\n args.update(kwargs)\n return _call_system_api(**args)\n\n async def connect(self, **kwargs: Any) -> tconn.Connection:\n conn_args = self.get_connect_args(**kwargs)\n return await tconn.async_connect_test_client(**conn_args)\n\n async def connect_pg(self, **kwargs: Any) -> asyncpg.Connection:\n import asyncpg\n\n conn_args = self.get_connect_args(**kwargs)\n return await asyncpg.connect(\n host=conn_args['host'],\n port=conn_args['port'],\n user=conn_args['user'],\n password=conn_args['password'],\n ssl='require'\n )\n\n async def connect_test_protocol(self, **kwargs):\n conn_args = self.get_connect_args(**kwargs)\n conn = await test_protocol.new_connection(**conn_args)\n await conn.connect()\n return conn\n\n\nclass _EdgeDBServer:\n\n proc: Optional[asyncio.Process]\n\n def __init__(\n self,\n *,\n bind_addrs: tuple[str, ...] = ('localhost',),\n bootstrap_command: Optional[str],\n auto_shutdown_after: Optional[int],\n adjacent_to: Optional[tconn.Connection],\n max_allowed_connections: Optional[int],\n compiler_pool_size: int,\n compiler_pool_mode: Optional[edgedb_args.CompilerPoolMode] = None,\n debug: bool,\n backend_dsn: Optional[str] = None,\n data_dir: Optional[str] = None,\n runstate_dir: Optional[str] = None,\n reset_auth: Optional[bool] = None,\n tenant_id: Optional[str] = None,\n security: edgedb_args.ServerSecurityMode,\n default_auth_method: Optional[\n edgedb_args.ServerAuthMethod | edgedb_args.ServerAuthMethods\n ] = None,\n binary_endpoint_security: Optional[\n edgedb_args.ServerEndpointSecurityMode] = None,\n http_endpoint_security: Optional[\n edgedb_args.ServerEndpointSecurityMode] = None, # see __aexit__\n enable_backend_adaptive_ha: bool = False,\n ignore_other_tenants: bool = False,\n readiness_state_file: Optional[str] = None,\n tls_cert_file: Optional[os.PathLike] = None,\n tls_key_file: Optional[os.PathLike] = None,\n tls_cert_mode: edgedb_args.ServerTlsCertMode = (\n edgedb_args.ServerTlsCertMode.SelfSigned),\n tls_client_ca_file: Optional[os.PathLike] = None,\n jws_key_file: Optional[os.PathLike] = None,\n jwt_sub_allowlist_file: Optional[os.PathLike] = None,\n jwt_revocation_list_file: Optional[os.PathLike] = None,\n multitenant_config: Optional[str] = None,\n config_file: Optional[os.PathLike] = None,\n default_branch: Optional[str] = None,\n env: Optional[dict[str, str]] = None,\n extra_args: Optional[list[str]] = None,\n net_worker_mode: Optional[str] = None,\n password: Optional[str] = None,\n ) -> None:\n self.bind_addrs = bind_addrs\n self.auto_shutdown_after = auto_shutdown_after\n self.bootstrap_command = bootstrap_command\n self.adjacent_to = adjacent_to\n self.max_allowed_connections = max_allowed_connections\n self.compiler_pool_size = compiler_pool_size\n self.compiler_pool_mode = compiler_pool_mode\n self.debug = debug\n self.backend_dsn = backend_dsn\n self.data_dir = data_dir\n self.runstate_dir = runstate_dir\n self.reset_auth = reset_auth\n self.tenant_id = tenant_id\n self.proc = None\n self.data = None\n self.security = security\n self.default_auth_method = default_auth_method\n self.binary_endpoint_security = binary_endpoint_security\n self.http_endpoint_security = http_endpoint_security\n self.enable_backend_adaptive_ha = enable_backend_adaptive_ha\n self.ignore_other_tenants = ignore_other_tenants\n self.readiness_state_file = readiness_state_file\n self.tls_cert_file = tls_cert_file\n self.tls_key_file = tls_key_file\n self.tls_cert_mode = tls_cert_mode\n self.tls_client_ca_file = tls_client_ca_file\n self.jws_key_file = jws_key_file\n self.jwt_sub_allowlist_file = jwt_sub_allowlist_file\n self.jwt_revocation_list_file = jwt_revocation_list_file\n self.multitenant_config = multitenant_config\n self.config_file = config_file\n self.default_branch = default_branch\n self.env = env\n self.extra_args = extra_args\n self.net_worker_mode = net_worker_mode\n self.password = password\n\n async def wait_for_server_readiness(self, stream: asyncio.StreamReader):\n while True:\n line = await stream.readline()\n if self.debug:\n print(line.decode())\n if not line:\n raise RuntimeError(\"Gel server terminated\")\n if line.startswith(b'READY='):\n break\n\n _, _, dataline = line.decode().partition('=')\n return json.loads(dataline)\n\n async def kill_process(self, proc: asyncio.Process):\n proc.terminate()\n try:\n await asyncio.wait_for(proc.wait(), timeout=60)\n except TimeoutError:\n proc.kill()\n\n async def _shutdown(self, exc: Optional[Exception] = None):\n if self.proc is None:\n return\n\n if self.proc.returncode is None:\n if self.auto_shutdown_after is not None and exc is None:\n try:\n await asyncio.wait_for(self.proc.wait(), timeout=60 * 5)\n except TimeoutError:\n self.proc.kill()\n raise AssertionError(\n 'server did not auto-shutdown in 5 minutes')\n else:\n await self.kill_process(self.proc)\n\n # asyncio, hello?\n # Workaround SubprocessProtocol.__del__ weirdly\n # complaining that loop is closed.\n self.proc._transport.close()\n\n self.proc = None\n\n async def __aenter__(self):\n status_r, status_w = socket.socketpair()\n\n cmd = [\n sys.executable, '-I', '-m', 'edb.server.main',\n '--port', 'auto',\n '--testmode',\n '--emit-server-status', f'fd://{status_w.fileno()}',\n '--compiler-pool-size', str(self.compiler_pool_size),\n '--tls-cert-mode', str(self.tls_cert_mode),\n '--jose-key-mode', 'generate',\n ]\n\n if self.compiler_pool_mode is not None:\n cmd.extend(('--compiler-pool-mode', self.compiler_pool_mode.value))\n\n for addr in self.bind_addrs:\n cmd.extend(('--bind-address', addr))\n\n reset_auth = self.reset_auth\n\n cmd.extend(['--log-level', 'd' if self.debug else 's'])\n if self.max_allowed_connections is not None:\n cmd.extend([\n '--max-backend-connections', str(self.max_allowed_connections),\n ])\n if self.backend_dsn is not None:\n cmd.extend([\n '--backend-dsn', self.backend_dsn,\n ])\n elif self.adjacent_to is not None:\n settings = self.adjacent_to.get_settings()\n pgdsn = settings.get('pgdsn')\n if pgdsn is None:\n raise RuntimeError('test requires devmode to access pgdsn')\n cmd += [\n '--backend-dsn', pgdsn.decode('utf-8')\n ]\n elif self.multitenant_config:\n cmd += ['--multitenant-config-file', self.multitenant_config]\n elif self.data_dir:\n cmd += ['--data-dir', self.data_dir]\n else:\n cmd += ['--temp-dir']\n\n if reset_auth is None:\n reset_auth = True\n\n if not reset_auth:\n password = self.password\n bootstrap_command = ''\n else:\n password = secrets.token_urlsafe()\n bootstrap_command = f\"\"\"\\\n ALTER ROLE admin {{\n SET password := '{password}';\n }};\n \"\"\"\n\n if self.bootstrap_command is not None:\n bootstrap_command += self.bootstrap_command\n\n if bootstrap_command:\n cmd += ['--bootstrap-command', bootstrap_command]\n\n if self.default_branch is not None:\n cmd += ['--default-branch', self.default_branch]\n\n if self.auto_shutdown_after is not None:\n cmd += ['--auto-shutdown-after', str(self.auto_shutdown_after)]\n\n if self.runstate_dir:\n cmd += ['--runstate-dir', self.runstate_dir]\n\n if self.tenant_id:\n cmd += ['--tenant-id', self.tenant_id]\n\n if self.security:\n cmd += ['--security', str(self.security)]\n\n if self.default_auth_method:\n cmd += ['--default-auth-method', str(self.default_auth_method)]\n\n if self.binary_endpoint_security:\n cmd += ['--binary-endpoint-security',\n str(self.binary_endpoint_security)]\n\n if self.http_endpoint_security:\n cmd += ['--http-endpoint-security',\n str(self.http_endpoint_security)]\n\n if self.enable_backend_adaptive_ha:\n cmd += ['--enable-backend-adaptive-ha']\n\n if self.ignore_other_tenants:\n cmd += ['--ignore-other-tenants']\n\n if self.tls_cert_file:\n cmd += ['--tls-cert-file', self.tls_cert_file]\n\n if self.tls_key_file:\n cmd += ['--tls-key-file', self.tls_key_file]\n\n if self.tls_client_ca_file:\n cmd += ['--tls-client-ca-file', str(self.tls_client_ca_file)]\n\n if self.readiness_state_file:\n cmd += ['--readiness-state-file', self.readiness_state_file]\n\n if self.jws_key_file:\n cmd += ['--jws-key-file', str(self.jws_key_file)]\n\n if self.jwt_sub_allowlist_file:\n cmd += ['--jwt-sub-allowlist-file', self.jwt_sub_allowlist_file]\n\n if self.jwt_revocation_list_file:\n cmd += ['--jwt-revocation-list-file',\n self.jwt_revocation_list_file]\n\n if self.config_file:\n cmd += ['--config-file', self.config_file]\n\n if not self.multitenant_config:\n cmd += ['--instance-name=localtest']\n\n if self.net_worker_mode:\n cmd += ['--net-worker-mode', self.net_worker_mode]\n\n if self.extra_args:\n cmd.extend(self.extra_args)\n\n if self.debug:\n print(\n f'Starting Gel cluster with the following params:\\n'\n f'{\" \".join(shlex.quote(c) for c in cmd)}'\n )\n\n env = os.environ.copy()\n if self.env:\n env.update(self.env)\n env.pop(\"EDGEDB_SERVER_MULTITENANT_CONFIG_FILE\", None)\n\n stat_reader, stat_writer = await asyncio.open_connection(sock=status_r)\n\n self.proc: asyncio.Process = await asyncio.create_subprocess_exec(\n *cmd,\n env=env,\n stdout=subprocess.PIPE,\n stderr=subprocess.STDOUT,\n pass_fds=(status_w.fileno(),),\n )\n\n status_task = asyncio.create_task(\n asyncio.wait_for(\n self.wait_for_server_readiness(stat_reader),\n timeout=240,\n ),\n )\n\n output = b''\n\n async def read_stdout():\n nonlocal output\n # Tee the log temporarily to a tempfile that exists as long as the\n # test is running. This helps debug hanging tests.\n with tempfile.NamedTemporaryFile(\n mode='w+t',\n prefix='edgedb-test-log-') as temp_file:\n if self.debug:\n print(f\"Logging to {temp_file.name}\")\n while True:\n line = await self.proc.stdout.readline()\n if not line:\n break\n output += line\n temp_file.write(line.decode(errors='ignore'))\n if self.debug:\n print(line.decode(errors='ignore'), end='')\n\n stdout_task = asyncio.create_task(read_stdout())\n\n try:\n _, pending = await asyncio.wait(\n [\n status_task,\n asyncio.create_task(self.proc.wait()),\n ],\n return_when=asyncio.FIRST_COMPLETED,\n )\n except (Exception, asyncio.CancelledError):\n try:\n await self._shutdown()\n finally:\n raise\n finally:\n stat_writer.close()\n status_w.close()\n\n if pending:\n for task in pending:\n if not task.done():\n task.cancel()\n\n await asyncio.wait(pending, timeout=10)\n\n if self.proc.returncode is not None:\n await stdout_task\n raise edgedb_cluster.ClusterError(output.decode(errors='ignore'))\n else:\n assert status_task.done()\n data = status_task.result()\n\n return _EdgeDBServerData(\n host='localhost',\n port=data['port'],\n password=password,\n server_data=data,\n tls_cert_file=data['tls_cert_file'],\n pid=self.proc.pid,\n )\n\n async def __aexit__(self, exc_type, exc, tb):\n try:\n if (\n (\n self.http_endpoint_security\n is edgedb_args.ServerEndpointSecurityMode.Optional\n )\n and self.data is not None\n and self.auto_shutdown_after is None\n ):\n # It's a good idea to test most of the ad-hoc test clusters\n # for any errors in background tasks, as such tests usually\n # test the functionality that involves notifications and\n # other async events.\n metrics = _fetch_metrics('127.0.0.1', self.data['port'])\n errors = _extract_background_errors(metrics)\n if errors:\n raise AssertionError(\n 'server terminated with unexpected ' +\n 'background errors\\n\\n' +\n errors\n )\n finally:\n await self._shutdown(exc)\n\n\ndef start_edgedb_server(\n *,\n bind_addrs: tuple[str, ...] = ('localhost',),\n auto_shutdown_after: Optional[int]=None,\n bootstrap_command: Optional[str]=None,\n max_allowed_connections: Optional[int]=5,\n compiler_pool_size: int=2,\n compiler_pool_mode: Optional[edgedb_args.CompilerPoolMode] = None,\n adjacent_to: Optional[tconn.Connection]=None,\n debug: bool=debug.flags.server,\n backend_dsn: Optional[str] = None,\n runstate_dir: Optional[str] = None,\n data_dir: Optional[str] = None,\n reset_auth: Optional[bool] = None,\n tenant_id: Optional[str] = None,\n security: edgedb_args.ServerSecurityMode = (\n edgedb_args.ServerSecurityMode.Strict),\n default_auth_method: Optional[\n edgedb_args.ServerAuthMethod | edgedb_args.ServerAuthMethods\n ] = None,\n binary_endpoint_security: Optional[\n edgedb_args.ServerEndpointSecurityMode] = None,\n http_endpoint_security: Optional[\n edgedb_args.ServerEndpointSecurityMode] = None,\n enable_backend_adaptive_ha: bool = False,\n ignore_other_tenants: bool = False,\n readiness_state_file: Optional[str] = None,\n tls_cert_file: Optional[os.PathLike] = None,\n tls_key_file: Optional[os.PathLike] = None,\n tls_cert_mode: edgedb_args.ServerTlsCertMode = (\n edgedb_args.ServerTlsCertMode.SelfSigned),\n tls_client_ca_file: Optional[os.PathLike] = None,\n jws_key_file: Optional[os.PathLike] = None,\n jwt_sub_allowlist_file: Optional[os.PathLike] = None,\n jwt_revocation_list_file: Optional[os.PathLike] = None,\n multitenant_config: Optional[str] = None,\n config_file: Optional[os.PathLike] = None,\n env: Optional[dict[str, str]] = None,\n extra_args: Optional[list[str]] = None,\n default_branch: Optional[str] = None,\n net_worker_mode: Optional[str] = None,\n force_new: bool = False, # True for ignoring multitenant config env\n):\n if (not devmode.is_in_dev_mode() or adjacent_to) and not runstate_dir:\n if backend_dsn or adjacent_to:\n import traceback\n # We don't want to implicitly \"fix the issue\" for the test author\n print('WARNING: starting a Gel server with the default '\n 'runstate_dir; the test is likely to fail or hang. '\n 'Consider specifying the runstate_dir parameter.')\n print('\\n'.join(traceback.format_stack(limit=5)))\n\n password = None\n if mt_conf := os.environ.get(\"EDGEDB_SERVER_MULTITENANT_CONFIG_FILE\"):\n if multitenant_config is None and max_allowed_connections == 10:\n if not any(\n (\n adjacent_to,\n data_dir,\n backend_dsn,\n compiler_pool_mode,\n default_branch,\n force_new,\n )\n ):\n multitenant_config = mt_conf\n max_allowed_connections = None\n password = 'test' # set in init_cluster() by test/runner.py\n\n params = locals()\n exclusives = [\n name\n for name in [\n \"adjacent_to\",\n \"data_dir\",\n \"backend_dsn\",\n \"multitenant_config\",\n ]\n if params[name]\n ]\n if len(exclusives) > 1:\n raise RuntimeError(\n \" and \".join(exclusives) + \" options are mutually exclusive\"\n )\n\n if not runstate_dir and data_dir:\n runstate_dir = data_dir\n\n return _EdgeDBServer(\n bind_addrs=bind_addrs,\n auto_shutdown_after=auto_shutdown_after,\n bootstrap_command=bootstrap_command,\n max_allowed_connections=max_allowed_connections,\n adjacent_to=adjacent_to,\n compiler_pool_size=compiler_pool_size,\n compiler_pool_mode=compiler_pool_mode,\n debug=debug,\n backend_dsn=backend_dsn,\n tenant_id=tenant_id,\n data_dir=data_dir,\n runstate_dir=runstate_dir,\n reset_auth=reset_auth,\n security=security,\n default_auth_method=default_auth_method,\n binary_endpoint_security=binary_endpoint_security,\n http_endpoint_security=http_endpoint_security,\n enable_backend_adaptive_ha=enable_backend_adaptive_ha,\n ignore_other_tenants=ignore_other_tenants,\n readiness_state_file=readiness_state_file,\n tls_cert_file=tls_cert_file,\n tls_key_file=tls_key_file,\n tls_cert_mode=tls_cert_mode,\n tls_client_ca_file=tls_client_ca_file,\n jws_key_file=jws_key_file,\n jwt_sub_allowlist_file=jwt_sub_allowlist_file,\n jwt_revocation_list_file=jwt_revocation_list_file,\n multitenant_config=multitenant_config,\n config_file=config_file,\n env=env,\n extra_args=extra_args,\n default_branch=default_branch,\n net_worker_mode=net_worker_mode,\n password=password,\n )\n\n\ndef get_cases_by_shard(cases, selected_shard, total_shards, verbosity, stats):\n if total_shards <= 1:\n return cases\n\n selected_shard -= 1 # starting from 0\n new_test_est = 0.1 # default estimate if test is not found in stats\n new_setup_est = 1 # default estimate if setup is not found in stats\n\n # For logging\n total_tests = 0\n selected_tests = 0\n total_est = 0\n selected_est = 0\n\n # Priority queue of tests grouped by setup script ordered by estimated\n # running time of the groups. Order of tests within cases is preserved.\n tests_by_setup = []\n\n # Priority queue of individual tests ordered by estimated running time.\n tests_with_est = []\n\n # Prepare the source heaps\n setup_count = 0\n for case, tests in cases.items():\n # Extract zREPEAT tests and attach them to their first runs\n combined = {}\n for test in tests:\n test_name = str(test)\n orig_name = test_name.replace('test_zREPEAT', 'test')\n if orig_name == test_name:\n if test_name in combined:\n combined[test_name] = (test, *combined[test_name])\n else:\n combined[test_name] = (test,)\n else:\n if orig_name in combined:\n combined[orig_name] = (*combined[orig_name], test)\n else:\n combined[orig_name] = (test,)\n\n setup_script_getter = getattr(case, 'get_setup_script', None)\n if setup_script_getter and combined:\n tests_per_setup = []\n est_per_setup = setup_est = stats.get(\n 'setup::' + case.get_database_name(), (new_setup_est, 0),\n )[0]\n for test_name, test in combined.items():\n total_tests += len(test)\n est = stats.get(test_name, (new_test_est, 0))[0] * len(test)\n est_per_setup += est\n tests_per_setup.append((est, test))\n heapq.heappush(\n tests_by_setup,\n (-est_per_setup, setup_count, setup_est, tests_per_setup),\n )\n setup_count += 1\n total_est += est_per_setup\n else:\n for test_name, test in combined.items():\n total_tests += len(test)\n est = stats.get(test_name, (new_test_est, 0))[0] * len(test)\n total_est += est\n heapq.heappush(tests_with_est, (-est, total_tests, test))\n\n target_est = total_est / total_shards # target running time of one shard\n shards_est = [(0, shard, set()) for shard in range(total_shards)]\n cases = {} # output\n setup_to_alloc = set(range(setup_count)) # tracks first run of each setup\n\n # Assign per-setup tests first\n while tests_by_setup:\n remaining_est, setup_id, setup_est, tests = heapq.heappop(\n tests_by_setup,\n )\n est_acc, current, setups = heapq.heappop(shards_est)\n\n # Add setup time\n if setup_id not in setups:\n setups.add(setup_id)\n est_acc += setup_est\n if current == selected_shard:\n selected_est += setup_est\n if setup_id in setup_to_alloc:\n setup_to_alloc.remove(setup_id)\n else:\n # This means one more setup for the overall test run\n target_est += setup_est / total_shards\n\n # Add as much tests from this group to current shard as possible\n while tests:\n est, test = tests.pop(0)\n est_acc += est # est is a positive number\n remaining_est += est # remaining_est is a negative number\n\n if current == selected_shard:\n # Add the test to the result\n _add_test(cases, test)\n selected_tests += len(test)\n selected_est += est\n\n if est_acc >= target_est and -remaining_est > setup_est * 2:\n # Current shard is full and the remaining tests would take more\n # time than their setup, then add the tests back to the heap so\n # that we could add them to another shard\n heapq.heappush(\n tests_by_setup,\n (remaining_est, setup_id, setup_est, tests),\n )\n break\n\n heapq.heappush(shards_est, (est_acc, current, setups))\n\n # Assign all non-setup tests, but leave the last shard for everything else\n setups = set()\n while tests_with_est and len(shards_est) > 1:\n est, _, test = heapq.heappop(tests_with_est) # est is negative\n est_acc, current, setups = heapq.heappop(shards_est)\n est_acc -= est\n\n if current == selected_shard:\n # Add the test to the result\n _add_test(cases, test)\n selected_tests += len(test)\n selected_est -= est\n\n if est_acc >= target_est:\n # The current shard is full\n if current == selected_shard:\n # End early if the selected shard is full\n break\n else:\n # Only add the current shard back to the heap if it's not full\n heapq.heappush(shards_est, (est_acc, current, setups))\n\n else:\n # Add all the remaining tests to the first remaining shard if any\n while shards_est:\n est_acc, current, setups = heapq.heappop(shards_est)\n if current == selected_shard:\n for est, _, test in tests_with_est:\n _add_test(cases, test)\n selected_tests += len(test)\n selected_est -= est\n break\n tests_with_est.clear() # should always be empty already here\n\n if verbosity >= 1:\n print(f'Running {selected_tests}/{total_tests} tests for shard '\n f'#{selected_shard + 1} out of {total_shards} shards, '\n f'estimate: {int(selected_est / 60)}m {int(selected_est % 60)}s'\n f' / {int(total_est / 60)}m {int(total_est % 60)}s, '\n f'{len(setups)}/{setup_count} databases to setup.')\n return _merge_results(cases)\n\n\ndef find_available_port() -> int:\n with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:\n sock.bind((\"localhost\", 0))\n return sock.getsockname()[1]\n\n\ndef _needs_factoring(weakly):\n def decorator(f):\n async def g(self, *args, **kwargs):\n if self.NO_FACTOR and not weakly:\n with self.assertRaisesRegex(Exception, ''):\n await f(self, *args, **kwargs)\n elif self.WARN_FACTOR:\n with self.assertRaisesRegex(\n edgedb.InvalidReferenceError, 'attempting to factor out'\n ):\n await f(self, *args, **kwargs)\n\n else:\n await f(self, *args, **kwargs)\n\n return g\n return decorator\n\n\n@contextlib.asynccontextmanager\nasync def temp_file_with(data: bytes):\n with tempfile.NamedTemporaryFile() as f:\n f.write(data)\n f.flush()\n yield f\n\n\nneeds_factoring = _needs_factoring(weakly=False)\nneeds_factoring_weakly = _needs_factoring(weakly=True)\n" }, { "path": "edb/tools/__init__.py", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2016-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\n# DO NOT ADD ANYTHING TO THIS FILE:\n# This package contains tools like mypy plugins that\n# when loaded should not trigger any imports.\n" }, { "path": "edb/tools/__main__.py", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2016-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\n\"\"\"Stub to allow invoking `edb` as `python -m edb.tools`.\"\"\"\n\nfrom __future__ import annotations\n\nimport sys\n\nfrom edb.tools import edb\n\n\nif __name__ == '__main__':\n sys.exit(edb.edbcommands(prog_name='edb'))\n" }, { "path": "edb/tools/ast_inheritance_graph.py", "content": "# Generates an inheritance graph of Python classes.\n#\n# Usage:\n# $ edb ast-inheritance-graph | fdp -T svg -o ast-fdp.svg\n#\n# Requirements:\n# - graphviz\n\nimport typing\nimport dataclasses\nimport enum\n\nimport click\n\nfrom edb.edgeql import ast as qlast\nfrom edb.ir import ast as irast\nfrom edb.pgsql import ast as pgast\nfrom edb.tools.edb import edbcommands\n\n\nclass ASTModule(str, enum.Enum):\n ql = \"ql\"\n ir = \"ir\"\n pg = \"pg\"\n\n\n@dataclasses.dataclass()\nclass ASTClass:\n name: str\n typ: typing.Any\n bases: set[str]\n children: set[str]\n\n\n@edbcommands.command(\"ast-inheritance-graph\")\n@click.argument('ast', type=click.Choice(ASTModule)) # type: ignore\ndef main(ast: ASTModule) -> None:\n ast_mod: typing.Any\n if ast == ASTModule.ql:\n ast_mod = qlast\n elif ast == ASTModule.ir:\n ast_mod = irast\n elif ast == ASTModule.pg:\n ast_mod = pgast\n else:\n raise AssertionError()\n\n # discover all nodes\n ast_classes: dict[str, ASTClass] = {}\n for name, typ in ast_mod.__dict__.items():\n if not isinstance(typ, type):\n continue\n\n if not issubclass(typ, ast_mod.Base) or name in {\n 'Base',\n 'ImmutableBase',\n }:\n continue\n\n if typ.__rust_ignore__: # type: ignore\n continue\n\n # re-run field collection to correctly handle forward-references\n typ = typ._collect_direct_fields() # type: ignore\n\n ast_classes[typ.__name__] = ASTClass(\n name=name,\n typ=typ,\n children=set(),\n bases=set(),\n )\n\n for ast_class in ast_classes.values():\n for base in ast_class.typ.__bases__:\n if base.__name__ not in ast_classes:\n continue\n ast_class.bases.add(base.__name__)\n ast_classes[base.__name__].children.add(ast_class.name)\n\n inheritance_graph(ast_classes)\n enum_graph(ast_classes)\n\n\ndef inheritance_graph(ast_classes: dict[str, ASTClass]):\n print('digraph I {')\n for ast_class in ast_classes.values():\n if ast_class.typ.__abstract_node__:\n print(f' {ast_class.name} [color = red];')\n for base in ast_class.bases:\n print(f' {ast_class.name} -> {base};')\n\n print('}')\n\n\ndef enum_graph(ast_classes: dict[str, ASTClass]):\n print('digraph M {')\n\n def dfs(node, start):\n ast_class = ast_classes[node]\n if ast_class.typ.__abstract_node__:\n print(f' {node}_{start} [color = red];')\n\n for child in ast_class.children:\n print(f' {node}_{start} -> {child}_{start};')\n dfs(child, start)\n\n for ast_class in ast_classes.values():\n if len(ast_class.bases) != 0 or len(ast_class.children) == 0:\n continue\n dfs(ast_class.name, ast_class.name)\n\n print('}')\n" }, { "path": "edb/tools/cli.py", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2021-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nimport subprocess\nimport sys\n\nimport click\n\nfrom edb import cli as rustcli\nfrom edb.tools.edb import edbcommands\n\n\n@edbcommands.command('cli',\n add_help_option=False,\n context_settings=dict(ignore_unknown_options=True))\n@click.argument('args', nargs=-1, type=click.UNPROCESSED)\ndef cli(args: tuple[str, ...]):\n \"\"\"Run edgedb CLI with `-H localhost`.\"\"\"\n\n args_list = _ensure_linked(args)\n\n if (\n '--wait-until-available' not in args_list and\n not any('--wait-until-available=' in a for a in args_list)\n ):\n args_list += ['--wait-until-available', '60s']\n\n sys.exit(rustcli.rustcli(args=[sys.argv[0], *args_list]))\n\n\n@edbcommands.command('ui',\n add_help_option=False,\n context_settings=dict(ignore_unknown_options=True))\n@click.argument('args', nargs=-1, type=click.UNPROCESSED)\ndef ui(args: tuple[str, ...]):\n \"\"\"Run edgedb GUI with `-H localhost`.\"\"\"\n\n _ensure_linked(args)\n subprocess.check_call(\n [\n sys.executable,\n \"-I\",\n \"-m\",\n \"edb.cli\",\n \"ui\",\n \"--instance=_localdev\",\n ],\n )\n\n\ndef _ensure_linked(args: tuple[str, ...]) -> list[str]:\n\n if (\n '--host' not in args and\n not any(a.startswith('-H') for a in args) and\n not any(a.startswith('--host=') for a in args) and\n '--port' not in args and\n not any(a.startswith('-P') for a in args) and\n not any(a.startswith('--port=') for a in args) and\n '--instance' not in args and\n not any(a.startswith('-I') for a in args) and\n not any(a.startswith('--instance=') for a in args)\n ):\n subprocess.check_call([\n sys.executable,\n \"-I\",\n \"-m\",\n \"edb.cli\",\n \"instance\",\n \"link\",\n \"--host=localhost\",\n \"--non-interactive\",\n \"--trust-tls-cert\",\n \"--overwrite\",\n \"--quiet\",\n \"_localdev\",\n ])\n\n return list(args) + ['-I', '_localdev']\n else:\n return list(args)\n" }, { "path": "edb/tools/config.py", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2020-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nfrom __future__ import annotations\n\n\nimport click\n\nfrom edb import buildmeta\nfrom edb.common import devmode\nfrom edb.tools.edb import edbcommands\n\n\n@edbcommands.command(\"config\")\n@click.option(\n \"--make-include\",\n is_flag=True,\n help='Print path to the include file for extension Makefiles',\n)\n@click.option(\n \"--pg-config\",\n is_flag=True,\n help='Print path to bundled pg_config',\n)\ndef config(make_include: bool, pg_config: bool) -> None:\n '''Query certain parameters about an edgedb environment'''\n if make_include:\n share = buildmeta.get_extension_dir_path()\n base = share.parent.parent.parent\n # XXX: It should not be here.\n if not devmode.is_in_dev_mode():\n base = base / 'share'\n mk = (\n base / 'tests' / 'extension-testing' / 'exts.mk'\n )\n print(mk)\n if pg_config:\n print(buildmeta.get_pg_config_path())\n" }, { "path": "edb/tools/dflags.py", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2019-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nfrom __future__ import annotations\n\nfrom edb.common import debug\nfrom edb.tools.edb import edbcommands\n\n\n@edbcommands.command('dflags')\ndef gen_types():\n \"\"\"Print available debug flags.\"\"\"\n\n for flag in debug.flags:\n print(f'env EDGEDB_DEBUG_{flag.name.upper()}=1')\n print(f' {flag.doc}\\n')\n" }, { "path": "edb/tools/docs/__init__.py", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2018-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nfrom __future__ import annotations\n\nfrom docutils import nodes as d_nodes\nfrom docutils.parsers import rst as d_rst\nfrom sphinx import addnodes as s_nodes\nfrom sphinx import transforms as s_transforms\nfrom sphinx.domains.index import IndexDirective\n\nfrom . import edb\nfrom . import cli\nfrom . import eql\nfrom . import js\nfrom . import sdl\nfrom . import graphql\nfrom . import go\nfrom . import shared\n\n\nclass ProhibitedNodeTransform(s_transforms.SphinxTransform):\n\n default_priority = 1 # before ReferencesResolver\n\n def apply(self):\n for bq in list(self.document.traverse(d_nodes.block_quote)):\n if not bq['classes'] or 'pull-quote' not in bq['classes']:\n raise shared.EdgeSphinxExtensionError(\n f'blockquote found: {bq.asdom().toxml()!r} in {bq.source};'\n f' Try using the \"pull-quote\" directive')\n else:\n bq.get('classes').remove('pull-quote')\n\n trs = list(self.document.traverse(d_nodes.title_reference))\n if trs:\n raise shared.EdgeSphinxExtensionError(\n f'title reference (single backticks quote) found: '\n f'{trs[0].asdom().toxml()!r} in {trs[0].source}; '\n f'perhaps you wanted to use double backticks?')\n\n\nclass VersionAdded(d_rst.Directive):\n\n has_content = True\n optional_arguments = 0\n required_arguments = 1\n\n def run(self):\n node = s_nodes.versionmodified()\n node['type'] = 'versionadded'\n node['version'] = self.arguments[0]\n self.state.nested_parse(self.content, self.content_offset, node)\n return [node]\n\n\nclass VersionChanged(d_rst.Directive):\n\n has_content = True\n optional_arguments = 0\n required_arguments = 1\n\n def run(self):\n node = s_nodes.versionmodified()\n node['type'] = 'versionchanged'\n node['version'] = self.arguments[0]\n self.state.nested_parse(self.content, self.content_offset, node)\n return [node]\n\n\nclass VersionedSection(d_rst.Directive):\n\n has_content = False\n optional_arguments = 0\n required_arguments = 0\n\n def run(self):\n node = d_nodes.container()\n node['versioned-section'] = True\n return [node]\n\n\nclass VersionedReplaceRole:\n\n def __call__(\n self, role, rawtext, text, lineno, inliner, options=None, content=None\n ):\n nodes = []\n if not text.startswith('_default:'):\n text = '_default:' + text\n for section in text.split(';'):\n parts = section.split(':', maxsplit=1)\n node = s_nodes.versionmodified()\n node['type'] = 'versionchanged'\n node['version'] = parts[0].strip()\n node += d_nodes.Text(parts[1].strip())\n nodes.append(node)\n return nodes, []\n\n\nclass APIIndex(IndexDirective):\n\n def run(self):\n nodes = super().run()\n nodes[0]['api-index'] = True\n return nodes\n\n\ndef setup(app):\n edb.setup_domain(app)\n cli.setup_domain(app)\n eql.setup_domain(app)\n js.setup_domain(app)\n sdl.setup_domain(app)\n graphql.setup_domain(app)\n go.setup_domain(app)\n\n app.add_directive('versionadded', VersionAdded, True)\n app.add_directive('versionchanged', VersionChanged, True)\n app.add_directive('code-block', shared.CodeBlock, True)\n app.add_directive('versioned-section', VersionedSection)\n app.add_directive('api-index', APIIndex)\n app.add_role('versionreplace', VersionedReplaceRole())\n\n app.add_transform(ProhibitedNodeTransform)\n" }, { "path": "edb/tools/docs/cli.py", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2018-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nfrom __future__ import annotations\n\nfrom typing import Any\n\nfrom edb.tools.pygments.edgeql import EdgeQLLexer\n\nimport pygments.lexers\n\nfrom sphinx import domains as s_domains\nfrom docutils.parsers.rst import directives as d_directives # type: ignore\n\nfrom . import shared\n\n\nclass CLISynopsisDirective(shared.CodeBlock):\n\n has_content = True\n optional_arguments = 0\n required_arguments = 0\n option_spec: dict[str, Any] = {\n 'version-lt': d_directives.unchanged_required\n }\n\n def run(self):\n self.arguments = ['cli-synopsis']\n return super().run()\n\n\nclass CLIDomain(s_domains.Domain):\n\n name = \"cli\"\n label = \"Command Line Interface\"\n\n directives = {\n 'synopsis': CLISynopsisDirective,\n }\n\n\ndef setup_domain(app):\n # Dummy lexers; the actual highlighting is implemented\n # in the edgedb.com website code.\n app.add_lexer(\"txt\", pygments.lexers.TextLexer)\n app.add_lexer(\"bash\", pygments.lexers.TextLexer)\n\n app.add_lexer(\"cli\", EdgeQLLexer)\n app.add_lexer(\"cli-synopsis\", EdgeQLLexer)\n\n app.add_role(\n 'cli:synopsis',\n shared.InlineCodeRole('cli-synopsis'))\n\n app.add_domain(CLIDomain)\n" }, { "path": "edb/tools/docs/edb.py", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2018-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nfrom __future__ import annotations\n\nimport re\n\nfrom sphinx import domains as s_domains\nfrom docutils import nodes as d_nodes\nfrom docutils.parsers import rst as d_rst\nfrom docutils.parsers.rst import directives as d_directives # type: ignore\n\nfrom sphinx import transforms\n\n\nclass EDBYoutubeEmbed(d_rst.Directive):\n\n has_content = True\n optional_arguments = 0\n required_arguments = 1\n\n def run(self):\n node = d_nodes.container()\n node['youtube-video-id'] = self.arguments[0]\n self.state.nested_parse(self.content, self.content_offset, node)\n return [node]\n\n\nclass EDBCollapsed(d_rst.Directive):\n\n has_content = True\n optional_arguments = 0\n required_arguments = 0\n option_spec = {\n 'summary': d_directives.unchanged_required,\n }\n\n def run(self):\n node = d_nodes.container()\n node['collapsed_block'] = True\n if 'summary' in self.options:\n node['summary'] = self.options['summary']\n self.state.nested_parse(self.content, self.content_offset, node)\n return [node]\n\n\nclass EDBEnvironmentSwitcher(d_rst.Directive):\n\n has_content = False\n optional_arguments = 0\n required_arguments = 0\n\n def run(self):\n node = d_nodes.container()\n node['env-switcher'] = True\n return [node]\n\n\nclass EDBSplitSection(d_rst.Directive):\n\n has_content = True\n optional_arguments = 0\n required_arguments = 0\n\n def run(self):\n node = d_nodes.container()\n node['split-section'] = True\n self.state.nested_parse(self.content, self.content_offset, node)\n\n split_indexes = [\n index for index, child in enumerate(node.children)\n if isinstance(child, d_nodes.container) and child.get('split-point')\n ]\n if len(split_indexes) > 1:\n raise Exception(\n f'cannot have multiple edb:split-point\\'s in edb:split-section'\n )\n blocks = (\n node.children[:split_indexes[0]] if\n node.children[split_indexes[0]].get('code-above')\n else node.children[split_indexes[0] + 1:]\n ) if len(split_indexes) == 1 else [node.children[-1]]\n if len(blocks) < 1:\n raise Exception(\n f'no content found at end of edb:split-section block, '\n f'or before/after the edb:split-point in the edb:split-section'\n )\n return [node]\n\n\nclass EDBSplitPoint(d_rst.Directive):\n\n has_content = False\n optional_arguments = 1\n required_arguments = 0\n\n def run(self):\n node = d_nodes.container()\n node['split-point'] = True\n if len(self.arguments) > 0:\n if self.arguments[0] not in ['above', 'below']:\n raise Exception(\n f\"expected edb:split-point arg to be 'above', 'below' \"\n f\"or empty (defaults to 'below')\"\n )\n if self.arguments[0] == 'above':\n node['code-above'] = True\n return [node]\n\n\nclass GelDomain(s_domains.Domain):\n name = \"edb\"\n label = \"Gel\"\n\n directives = {\n 'collapsed': EDBCollapsed,\n 'youtube-embed': EDBYoutubeEmbed,\n 'env-switcher': EDBEnvironmentSwitcher,\n 'split-section': EDBSplitSection,\n 'split-point': EDBSplitPoint\n }\n\n\nclass GelSubstitutionTransform(transforms.SphinxTransform):\n default_priority = 0\n\n def apply(self):\n builder_name = \"html\"\n if hasattr(self.document.settings, 'env'):\n env = self.document.settings.env\n if env and hasattr(env, \"app\"):\n builder_name = env.app.builder.name\n\n # Traverse all substitution_reference nodes.\n for node in self.document.traverse(d_nodes.substitution_reference):\n nt = node.astext()\n if nt.lower() in {\n \"gel\", \"gel's\", \"edgedb\", \"gelcmd\", \".gel\", \"gel.toml\",\n \"gel-server\", \"geluri\", \"admin\", \"main\",\n \"branch\", \"branches\"\n }:\n if builder_name in {\"xml\", \"edge-xml\"}:\n if nt == \"gelcmd\":\n sub = d_nodes.literal(\n 'gel', 'gel',\n **{\n \"edb-gelcmd\": \"true\",\n \"edb-gelcmd-top\": \"true\",\n \"edb-substitution\": \"true\",\n }\n )\n elif nt == \"geluri\":\n sub = d_nodes.literal(\n 'gel', 'gel://',\n **{\n \"edb-geluri\": \"true\",\n \"edb-geluri-top\": \"true\",\n \"edb-substitution\": \"true\",\n }\n )\n else:\n sub = d_nodes.inline(\n nt, nt, **{\"edb-substitution\": \"true\"}\n )\n node.replace_self(sub)\n else:\n node.replace_self(d_nodes.Text(nt))\n\n\nclass GelCmdRole:\n\n def __call__(\n self, role, rawtext, text, lineno, inliner, options=None, content=None\n ):\n text = text.strip()\n text = re.sub(r'(\\n\\s*)+', \" \", text)\n if text.startswith(\"edgedb\"):\n fn = inliner.document.current_source\n raise Exception(\n f\"{fn}:{lineno} - :gelcmd:`{text}` - can't start with 'edgedb'\"\n )\n if text.startswith(\"gel \") or text == \"gel\":\n fn = inliner.document.current_source\n raise Exception(\n f\"{fn}:{lineno} - :gelcmd:`{text}` - can't start with 'gel'\"\n )\n text = f'gel {text}'\n node = d_nodes.literal(text, text)\n node[\"edb-gelcmd\"] = \"true\"\n node[\"edb-gelcmd-top\"] = \"false\"\n node[\"edb-substitution\"] = \"true\"\n return [node], []\n\n\nclass GelUriRole:\n\n def __call__(\n self, role, rawtext, text, lineno, inliner, options=None, content=None\n ):\n if text.startswith(\"edgedb://\"):\n fn = inliner.document.current_source\n raise Exception(\n f\"{fn}:{lineno} - :geluri:`{text}`\"\n f\" - can't start with 'edgedb://'\"\n )\n if text.startswith(\"gel://\"):\n fn = inliner.document.current_source\n raise Exception(\n f\"{fn}:{lineno} - :geluri:`{text}` - can't start with 'gel://'\"\n )\n text = f'gel://{text}'\n node = d_nodes.literal(text, text)\n node[\"edb-geluri\"] = \"true\"\n node[\"edb-geluri-top\"] = \"false\"\n node[\"edb-substitution\"] = \"true\"\n return [node], []\n\n\nclass DotGelRole:\n\n def __call__(\n self, role, rawtext, text, lineno, inliner, options=None, content=None\n ):\n if text.endswith(\".gel\") or text.endswith(\".esdl\"):\n fn = inliner.document.current_source\n raise Exception(\n f\"{fn}:{lineno} - :dotgel:`{text}`\"\n f\" - can't end with '.esdl' or '.gel'\"\n )\n text = f'{text}.gel'\n node = d_nodes.literal(text, text)\n node[\"edb-dotgel\"] = \"true\"\n node[\"edb-substitution\"] = \"true\"\n return [node], []\n\n\nclass GelEnvRole:\n\n def __call__(\n self, role, rawtext, text, lineno, inliner, options=None, content=None\n ):\n if (\n text.startswith(\"EDGEDB_\") or\n text.startswith(\"GEL_\") or\n text.startswith(\"_\")\n ):\n fn = inliner.document.current_source\n raise Exception(\n f\"{fn}:{lineno} - :gelenv:`{text}`\"\n f\" - can't start with 'EDGEDB_', 'GEL_', or '_'\"\n )\n text = f'GEL_{text}'\n node = d_nodes.literal(text, text)\n node[\"edb-gelenv\"] = \"true\"\n node[\"edb-substitution\"] = \"true\"\n return [node], []\n\n\ndef setup_domain(app):\n\n app.add_role('gelcmd', GelCmdRole())\n app.add_role('geluri', GelUriRole())\n app.add_role('dotgel', DotGelRole())\n app.add_role('gelenv', GelEnvRole())\n app.add_domain(GelDomain)\n app.add_transform(GelSubstitutionTransform)\n\n\ndef setup(app):\n setup_domain(app)\n" }, { "path": "edb/tools/docs/eql.py", "content": "#\n# This source file is part of the EdgeDB open source project.\n#\n# Copyright 2018-present MagicStack Inc. and the EdgeDB authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nr\"\"\"\n=====================================\n:eql: domain for EdgeQL documentation\n=====================================\n\n\nFunctions\n---------\n\nTo declare a function use a \".. eql:function::\" directive. A few\nthings must be defined:\n\n* Full function signature with a fully qualified name must be specified.\n\n* \":param $name: description:\" a short description of the $name parameter.\n $name must match the the name of the parameter in function's signature.\n If a parameter is anonymous, its number should be used instead (e.g. $1).\n\n* \":paramtype $name: type\": for every :param: there must be a\n corresponding :paramtype field. For example: \":paramtype $name: int64\"\n declares that the type of the $name parameter is `int64`. If a parameter\n has more than one valid types list them separated by \"or\":\n \":paramtype $name: int64 or str\".\n\n* :return: and :returntype: are similar to :param: and\n :paramtype: but lack parameter names. They must be used to document\n the return value of the function.\n\n* A few paragraphs and code samples. The first paragraph must\n be a single sentence no longer than 79 characters describing the\n function.\n\nExample:\n\n .. eql:function:: std::array_agg(SET OF any, $a: any) -> array\n\n :param $1: input set\n :paramtype $1: SET OF any\n\n :param $a: description of this param\n :paramtype $a: int64 or str\n\n :return: array made of input set elements\n :returntype: array\n\n Return the array made from all of the input set elements.\n\n The ordering of the input set will be preserved if specified.\n\n\nA function can be referenced from anywhere in the documentation by using\na \":eql:func:\" role. For instance:\n\n* \":eql:func:`array_agg`\";\n* \":eql:func:`std::array_agg`\";\n* or, \"look at this :eql:func:`fancy function `\".\n\n\nOperators\n---------\n\nUse \".. eql:operator::\" directive to declare an operator. Supported fields:\n\n* \":optype NAME: TYPE\" -- operand type.\n\nThe first argument of the directive must be a string in the following\nformat: \"OPERATOR_ID: OPERATOR SIGNATURE\". For instance, for a \"+\"\noperator it would be \"PLUS: A + B\":\n\n .. eql:operator:: PLUS: A + B\n\n :optype A: int64 or str or bytes\n :optype B: any\n :resulttype: any\n\n Arithmetic addition.\n\nTo reference an operator use the :eql:op: role along with OPERATOR_ID:\n\":eql:op:`plus`\" or \":eql:op:`+ `\". Operator ID is case-insensitive.\n\n\nStatements\n----------\n\nUse \":eql-statement:\" field for sections that describe a statement.\n\nA :eql-haswith: field should be used if the statement supports a WITH block.\n\nExample:\n\n SELECT\n ------\n\n :eql-statement:\n :eql-haswith:\n\n SELECT is used to select stuff.\n\n\n .. eql:synopsis::\n\n [WITH [MODULE name]]\n SELECT expr\n FILTER expr\n\n\n .. eql:clause:: FILTER: A FILTER B\n\n :paramtype A: any\n :paramtype B: SET OF any\n :returntype: any\n\n FILTER should be used to filter stuff.\n\n\n More paragraphs describing intricacies of SELECT go here...\n\n More paragraphs describing intricacies of SELECT go here...\n\n More paragraphs describing intricacies of SELECT go here...\n\nNotes:\n\n* To reference a statement use the \":eql:stmt:\" role. For instance:\n\n - :eql:stmt:`SELECT`\n - :eql:stmt:`my fav statement