[ { "path": ".dockerignore", "content": "# explicitly list the files needed by docker.\n*\n!package.json\n!yarn.lock\n!tsconfig-ext.json\n!tsconfig-prod.json\n!tsconfig.json\n!stubs\n!app\n!buildtools\n!static\n!bower_components\n!sandbox\n!plugins\n!test\n!ext\n**/_build\n" }, { "path": ".editorconfig", "content": "# EditorConfig is awesome: https://EditorConfig.org\n\n# top-most EditorConfig file\nroot = true\n\n# Unix-style newlines with a newline ending every file\n[*.{ts,js,py}]\nend_of_line = lf\ninsert_final_newline = true\ncharset = utf-8\n# indent with 2 spaces\nindent_style = space\nindent_size = 2\ntrim_trailing_whitespace = true\n" }, { "path": ".git-blame-ignore-revs", "content": "a7eb4d6e60c50375a835a5300b8e6beebcfb8422\n3a859ee5454d9c2b9600c9d9a3bd859d65d496bf\n9c9c45a6894e80748f0e337d4f164d40b503cae5\na3082e1f8b59b36154e721eb3d17c6504cf63bb8\n29541707086d4de9fbe8773f42e0f5f0f845ea54\n4faa64a1c0c194b32a804b2e562361ba197d8699\ne44e6217bd223873aef2f1aa2df32dc9c17a3abf\ne75c8ce28d9470ba5e92ee25aec65869a7051668\nb7e9ebed9dddd931a9bf33ae57dccb6423a7b6ae\na893706c5b9659157878318513f6d6cbe025e51d\n2f2cd7d60144486e67fc2fe0eaf20fcafcd50f21\n" }, { "path": ".github/FUNDING.yml", "content": "# These are supported funding model platforms\n\ngithub: gristlabs\n" }, { "path": ".github/ISSUE_TEMPLATE/00-bug-issue.yml", "content": "# Inspired by PeerTube templates:\n# https://github.com/Chocobozzz/PeerTube/blob/3d4d49a23eae71f3ce62cbbd7d93f07336a106b7/.github/ISSUE_TEMPLATE/00-bug-issue.yml\nname: 🐛 Bug Report\ndescription: Use this template for reporting a bug\nbody:\n - type: markdown\n attributes:\n value: |\n Thanks for taking time to fill out this bug report!\n Please search among past open/closed issues for a similar one beforehand:\n - https://github.com/gristlabs/grist-core/issues?q=\n - https://community.getgrist.com/\n\n - type: textarea\n attributes:\n label: Describe the current behavior\n\n - type: textarea\n attributes:\n label: Steps to reproduce\n value: |\n 1.\n 2.\n 3.\n\n - type: textarea\n attributes:\n label: Describe the expected behavior\n\n - type: checkboxes\n attributes:\n label: Where have you encountered this bug?\n options:\n - label: On [docs.getgrist.com](https://docs.getgrist.com)\n - label: On a self-hosted instance \n validations:\n required: true\n\n - type: textarea\n attributes:\n label: Instance information (when self-hosting only)\n description: In case you self-host, please share information above. You can discard any question you don't know the answer.\n value: |\n * Grist instance:\n * Version:\n * URL (if it's OK for you to share it):\n * Installation mode: docker/kubernetes/...\n * Architecture: single-worker/multi-workers\n\n * Browser name, version and platforms on which you could reproduce the bug:\n * Link to browser console log if relevant:\n * Link to server log if relevant:\n" }, { "path": ".github/ISSUE_TEMPLATE/10-installation-issue.yml", "content": "# Inspired by PeerTube templates:\n# https://github.com/Chocobozzz/PeerTube/blob/master/.github/ISSUE_TEMPLATE/10-installation-issue.yml\nname: 🛠️ Installation/Upgrade Issue\ndescription: Use this template for installation/upgrade issues\nbody:\n - type: markdown\n attributes:\n value: |\n Please check first the official documentation for self-hosting: https://support.getgrist.com/self-managed/\n\n - type: markdown\n attributes:\n value: |\n Please search among past open/closed issues for a similar one beforehand:\n - https://github.com/gristlabs/grist-core/issues?q=\n - https://community.getgrist.com/\n\n - type: textarea\n attributes:\n label: Describe the problem\n\n - type: textarea\n attributes:\n label: Additional information\n value: |\n * Grist version:\n * Grist instance URL:\n * SSO solution used and its version (if relevant):\n * S3 storage solution and its version (if relevant):\n * Docker version (if relevant):\n * NodeJS version (if relevant):\n * Redis version (if relevant):\n * PostgreSQL version (if relevant):\n" }, { "path": ".github/ISSUE_TEMPLATE/20-feature-request.yml", "content": "# Inspired by PeerTube templates:\n# https://github.com/Chocobozzz/PeerTube/blob/master/.github/ISSUE_TEMPLATE/30-feature-request.yml\n---\nname: ✨ Feature Request\ndescription: Use this template to ask for new features and suggest new ideas 💡\nbody:\n - type: markdown\n attributes:\n value: |\n Thanks for taking time to share your ideas!\n Please search among past open/closed issues for a similar one beforehand:\n - https://github.com/gristlabs/grist-core/issues?q=\n - https://community.getgrist.com/\n\n - type: textarea\n attributes:\n label: Describe the problem to be solved\n description: Provide a clear and concise description of what the problem is\n\n - type: textarea\n attributes:\n label: Describe the solution you would like\n description: Provide a clear and concise description of what you want to happen\n" }, { "path": ".github/ISSUE_TEMPLATE/config.yml", "content": "blank_issues_enabled: false\ncontact_links:\n - name: 🤷💻🤦 Question/Forum\n url: https://community.getgrist.com/\n about: You can ask and answer other questions here\n - name: 💬 Discord\n url: https://discord.com/invite/MYKpYQ3fbP\n about: Chat with us via Discord for quick Q/A here and sharing tips\n" }, { "path": ".github/PULL_REQUEST_TEMPLATE.md", "content": "## Context\n\n\n\n\n## Proposed solution\n\n\n\n## Related issues\n\n\n\n\n\n## Has this been tested?\n\n\n\n- [ ] 👍 yes, I added tests to the test suite\n- [ ] 💭 no, because this PR is a draft and still needs work\n- [ ] 🙅 no, because this is not relevant here\n- [ ] 🙋 no, because I need help \n\n## Screenshots / Screencasts\n\n\n" }, { "path": ".github/cla/individual-cla.md", "content": "# Individual Contributor License Agreement (\"Agreement\"), v1.0\n\n(Based on https://www.apache.org/licenses/icla.pdf by the Apache\nFoundation. Contact support@getgrist.com if you wish to execute a\nCorporate CLA.)\n\nThank you for your interest in contributing to software projects made\navailable by Grist Labs Inc (\"Grist\"). To clarify the intellectual\nproperty license granted with Contributions from any person or entity,\nGrist must have on file a signed Contributor License Agreement (\"CLA\")\nfrom each Contributor, indicating agreement with the license terms\nbelow. This agreement is for your protection as a Contributor as well\nas the protection of Grist and its users. It does not change your\nrights to use your own Contributions for any other purpose.\n\nContributions entirely composed of commits with authorship at\n`*.gouv.fr` domains fall outside the scope of this agreement.\n\nYou accept and agree to the following terms and conditions for Your\nContributions (present and future) that you submit to Grist. Except\nfor the license granted herein to Grist and recipients of software\ndistributed by Grist, You reserve all right, title, and interest in\nand to Your Contributions.\n\n1. Definitions.\n\n \"You\" (or \"Your\") shall mean the copyright owner or legal entity\n authorized by the copyright owner that is making this Agreement\n with Grist. For legal entities, the entity making a Contribution\n and all other entities that control, are controlled by, or are\n under common control with that entity are considered to be a single\n Contributor. For the purposes of this definition, \"control\" means\n (i) the power, direct or indirect, to cause the direction or\n management of such entity, whether by contract or otherwise, or\n (ii) ownership of fifty percent (50%) or more of the outstanding\n shares, or (iii) beneficial ownership of such entity.\n\n \"Contribution\" shall mean any original work of authorship,\n including any modifications or additions to an existing work, that\n is intentionally submitted by You to Grist for inclusion in, or\n documentation of, any of the products owned or managed by Grist\n (the \"Work\"). For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to Grist or its representatives, including but not limited to\n communication on electronic mailing lists, source code control\n systems, and issue tracking systems that are managed by, or on\n behalf of, Grist for the purpose of discussing and improving the\n Work, but excluding communication that is conspicuously marked or\n otherwise designated in writing by You as \"Not a Contribution.\"\n\n2. Grant of Copyright License.\n\n Subject to the terms and conditions of this Agreement, You hereby\n grant to Grist and to recipients of software distributed by Grist a\n perpetual, worldwide, non-exclusive, no-charge, royalty-free,\n irrevocable copyright license to reproduce, prepare derivative\n works of, publicly display, publicly perform, sublicense, and\n distribute Your Contributions and such derivative works.\n\n3. Grant of Patent License.\n\n Subject to the terms and conditions of this Agreement, You hereby\n grant to Grist and to recipients of software distributed by Grist a\n perpetual, worldwide, non-exclusive, no-charge, royalty-free,\n irrevocable (except as stated in this section) patent license to\n make, have made, use, offer to sell, sell, import, and otherwise\n transfer the Work, where such license applies only to those patent\n claims licensable by You that are necessarily infringed by Your\n Contribution(s) alone or by combination of Your Contribution(s)\n with the Work to which such Contribution(s) was submitted. If any\n entity institutes patent litigation against You or any other entity\n (including a cross-claim or counterclaim in a lawsuit) alleging\n that your Contribution, or the Work to which you have contributed,\n constitutes direct or contributory patent infringement, then any\n patent licenses granted to that entity under this Agreement for\n that Contribution or Work shall terminate as of the date such\n litigation is filed.\n\n4. You represent that you are legally entitled to grant the above\n license. If your employer(s) has rights to intellectual property\n that you create that includes your Contributions, you represent\n that you have received permission to make Contributions on behalf\n of that employer, that your employer has waived such rights for\n your Contributions to Grist, or that your employer has executed a\n separate Corporate CLA with Grist.\n\n5. You represent that each of Your Contributions is Your original\n creation (see section 7 for submissions on behalf of others). You\n represent that Your Contribution submissions include complete\n details of any third-party license or other restriction (including,\n but not limited to, related patents and trademarks) of which you\n are personally aware and which are associated with any part of Your\n Contributions.\n\n6. You are not expected to provide support for Your Contributions,\n except to the extent You desire to provide support. You may provide\n support for free, for a fee, or not at all. Unless required by\n applicable law or agreed to in writing, You provide Your\n Contributions on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS\n OF ANY KIND, either express or implied, including, without\n limitation, any warranties or conditions of TITLE,\n NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR\n PURPOSE.\n\n7. Should You wish to submit work that is not Your original creation,\n You may submit it to Grist separately from any Contribution,\n identifying the complete details of its source and of any license\n or other restriction (including, but not limited to, related\n patents, trademarks, and license agreements) of which you are\n personally aware, and conspicuously marking the work as \"Submitted\n on behalf of a third-party: [named here]\".\n\n8. You agree to notify Grist of any facts or circumstances of which\n you become aware that would make these representations inaccurate\n in any respect.\n" }, { "path": ".github/cla/signatures.json", "content": "{\n \"signedContributors\": [\n {\n \"name\": \"jordigh\",\n \"id\": 260143,\n \"comment_id\": 3053400623,\n \"created_at\": \"2025-07-09T17:12:17Z\",\n \"repoId\": 266033395,\n \"pullRequestNo\": 1682\n },\n {\n \"name\": \"georgegevoian\",\n \"id\": 85144792,\n \"comment_id\": 3070684406,\n \"created_at\": \"2025-07-14T19:16:24Z\",\n \"repoId\": 266033395,\n \"pullRequestNo\": 1705\n },\n {\n \"name\": \"scytacki\",\n \"id\": 86016,\n \"comment_id\": 3071744517,\n \"created_at\": \"2025-07-15T03:10:33Z\",\n \"repoId\": 266033395,\n \"pullRequestNo\": 1661\n },\n {\n \"name\": \"fflorent\",\n \"id\": 371705,\n \"comment_id\": 3079056343,\n \"created_at\": \"2025-07-16T15:06:22Z\",\n \"repoId\": 266033395,\n \"pullRequestNo\": 1683\n },\n {\n \"name\": \"manuhabitela\",\n \"id\": 221253,\n \"comment_id\": 3083417362,\n \"created_at\": \"2025-07-17T09:55:14Z\",\n \"repoId\": 266033395,\n \"pullRequestNo\": 1431\n },\n {\n \"name\": \"hexaltation\",\n \"id\": 31125573,\n \"comment_id\": 3083469227,\n \"created_at\": \"2025-07-17T10:11:19Z\",\n \"repoId\": 266033395,\n \"pullRequestNo\": 1699\n },\n {\n \"name\": \"paulfitz\",\n \"id\": 118367,\n \"comment_id\": 3084758085,\n \"created_at\": \"2025-07-17T16:53:25Z\",\n \"repoId\": 266033395,\n \"pullRequestNo\": 1711\n },\n {\n \"name\": \"Spoffy\",\n \"id\": 4805393,\n \"comment_id\": 3090253280,\n \"created_at\": \"2025-07-18T18:01:45Z\",\n \"repoId\": 266033395,\n \"pullRequestNo\": 1670\n },\n {\n \"name\": \"Anany-k\",\n \"id\": 13112955,\n \"comment_id\": 3095980233,\n \"created_at\": \"2025-07-21T09:55:52Z\",\n \"repoId\": 266033395,\n \"pullRequestNo\": 1716\n },\n {\n \"name\": \"vviers\",\n \"id\": 30295971,\n \"comment_id\": 3097269575,\n \"created_at\": \"2025-07-21T15:34:37Z\",\n \"repoId\": 266033395,\n \"pullRequestNo\": 1719\n },\n {\n \"name\": \"guillett\",\n \"id\": 1410356,\n \"comment_id\": 3113331623,\n \"created_at\": \"2025-07-24T12:43:18Z\",\n \"repoId\": 266033395,\n \"pullRequestNo\": 1691\n },\n {\n \"name\": \"ogui11aume\",\n \"id\": 31072389,\n \"comment_id\": 3118009823,\n \"created_at\": \"2025-07-25T14:21:07Z\",\n \"repoId\": 266033395,\n \"pullRequestNo\": 1653\n },\n {\n \"name\": \"mrdev023\",\n \"id\": 11292703,\n \"comment_id\": 3155373604,\n \"created_at\": \"2025-08-05T14:02:58Z\",\n \"repoId\": 266033395,\n \"pullRequestNo\": 1750\n },\n {\n \"name\": \"jonathanperret\",\n \"id\": 300823,\n \"comment_id\": 3233984142,\n \"created_at\": \"2025-08-28T15:30:02Z\",\n \"repoId\": 266033395,\n \"pullRequestNo\": 1778\n },\n {\n \"name\": \"berhalak\",\n \"id\": 11277225,\n \"comment_id\": 3249600375,\n \"created_at\": \"2025-09-03T14:53:49Z\",\n \"repoId\": 266033395,\n \"pullRequestNo\": 1807\n },\n {\n \"name\": \"Ajay-Satish-01\",\n \"id\": 71289526,\n \"comment_id\": 3260958924,\n \"created_at\": \"2025-09-06T05:28:40Z\",\n \"repoId\": 266033395,\n \"pullRequestNo\": 1818\n },\n {\n \"name\": \"dsagal\",\n \"id\": 1091143,\n \"comment_id\": 3293302321,\n \"created_at\": \"2025-09-15T17:52:16Z\",\n \"repoId\": 266033395,\n \"pullRequestNo\": 1841\n },\n {\n \"name\": \"tristanrobert\",\n \"id\": 19711088,\n \"comment_id\": 3297472676,\n \"created_at\": \"2025-09-16T10:20:26Z\",\n \"repoId\": 266033395,\n \"pullRequestNo\": 1840\n },\n {\n \"name\": \"ohemelaar\",\n \"id\": 19656762,\n \"comment_id\": 3326880813,\n \"created_at\": \"2025-09-24T07:05:33Z\",\n \"repoId\": 266033395,\n \"pullRequestNo\": 1830\n },\n {\n \"name\": \"nbush\",\n \"id\": 3422005,\n \"comment_id\": 3467956044,\n \"created_at\": \"2025-10-30T13:18:24Z\",\n \"repoId\": 266033395,\n \"pullRequestNo\": 1906\n },\n {\n \"name\": \"SimLV\",\n \"id\": 68837817,\n \"comment_id\": 3485625062,\n \"created_at\": \"2025-11-04T11:57:57Z\",\n \"repoId\": 266033395,\n \"pullRequestNo\": 1921\n },\n {\n \"name\": \"SleepyLeslie\",\n \"id\": 142967379,\n \"comment_id\": 3487547371,\n \"created_at\": \"2025-11-04T18:46:35Z\",\n \"repoId\": 266033395,\n \"pullRequestNo\": 1922\n },\n {\n \"name\": \"samchencode\",\n \"id\": 62081196,\n \"comment_id\": 3522061093,\n \"created_at\": \"2025-11-12T13:53:43Z\",\n \"repoId\": 266033395,\n \"pullRequestNo\": 1935\n },\n {\n \"name\": \"RapidShade\",\n \"id\": 20725534,\n \"comment_id\": 3533412593,\n \"created_at\": \"2025-11-14T15:54:39Z\",\n \"repoId\": 266033395,\n \"pullRequestNo\": 1945\n },\n {\n \"name\": \"cfpwastaken\",\n \"id\": 44261356,\n \"comment_id\": 3694780658,\n \"created_at\": \"2025-12-28T14:18:03Z\",\n \"repoId\": 266033395,\n \"pullRequestNo\": 2023\n },\n {\n \"name\": \"unknownconstant\",\n \"id\": 14999931,\n \"comment_id\": 3791801541,\n \"created_at\": \"2026-01-23T18:54:49Z\",\n \"repoId\": 266033395,\n \"pullRequestNo\": 2067\n },\n {\n \"name\": \"mikhailbogdan-droid\",\n \"id\": 238033187,\n \"comment_id\": 3823827850,\n \"created_at\": \"2026-01-30T13:42:28Z\",\n \"repoId\": 266033395,\n \"pullRequestNo\": 2085\n },\n {\n \"name\": \"Vortezz\",\n \"id\": 61989315,\n \"comment_id\": 3861607282,\n \"created_at\": \"2026-02-06T17:08:18Z\",\n \"repoId\": 266033395,\n \"pullRequestNo\": 2096\n },\n {\n \"name\": \"kosssi\",\n \"id\": 1135513,\n \"comment_id\": 4004185474,\n \"created_at\": \"2026-03-05T10:52:21Z\",\n \"repoId\": 266033395,\n \"pullRequestNo\": 2151\n },\n {\n \"name\": \"webash\",\n \"id\": 6205899,\n \"comment_id\": 4026041376,\n \"created_at\": \"2026-03-09T18:57:39Z\",\n \"repoId\": 266033395,\n \"pullRequestNo\": 2161\n },\n {\n \"name\": \"Gwabix\",\n \"id\": 50367170,\n \"comment_id\": 4060017206,\n \"created_at\": \"2026-03-14T08:23:49Z\",\n \"repoId\": 266033395,\n \"pullRequestNo\": 2178\n }\n ]\n}" }, { "path": ".github/workflows/cla.yml", "content": "# Workflow body from https://github.com/contributor-assistant/github-action\n\nname: \"CLA Assistant\"\non:\n issue_comment:\n types: [created]\n pull_request_target:\n types: [opened,closed,synchronize]\n\npermissions:\n actions: write\n contents: write\n pull-requests: write\n statuses: write\n\njobs:\n CLAAssistant:\n runs-on: ubuntu-latest\n steps:\n - name: \"CLA Assistant\"\n if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'\n uses: contributor-assistant/github-action@v2.6.1\n env:\n GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n with:\n path-to-signatures: '.github/cla/signatures.json'\n path-to-document: 'https://github.com/gristlabs/grist-core/blob/main/.github/cla/individual-cla.md'\n branch: 'main'\n allowlist: github-actions[bot],dependabot[bot]\n lock-pullrequest-aftermerge: false\n" }, { "path": ".github/workflows/docker.yml", "content": "name: Push Docker image\n\non:\n release:\n types: [published]\n # Allows you to run this workflow manually from the Actions tab\n workflow_dispatch:\n inputs:\n tag:\n description: \"Tag for the resulting images\"\n type: string\n required: True\n default: 'stable'\n\nenv:\n TAG: ${{ inputs.tag || 'stable' }}\n DOCKER_HUB_OWNER: ${{ vars.DOCKER_HUB_OWNER || github.repository_owner }}\n PLATFORMS: ${{ vars.PLATFORMS || 'linux/amd64,linux/arm64/v8' }}\n\njobs:\n push_to_registry:\n name: Push Docker images to Docker Hub\n runs-on: ubuntu-22.04\n strategy:\n matrix:\n image:\n # We build two images, `grist-oss` and `grist`.\n # See https://github.com/gristlabs/grist-core?tab=readme-ov-file#available-docker-images\n - name: \"grist-oss\"\n repo: \"grist-core\"\n - name: \"grist\"\n repo: \"grist-ee\"\n steps:\n - name: Free some space\n run: |\n sudo rm -rf /usr/share/dotnet\n sudo rm -rf /usr/local/lib/android\n sudo rm -rf /usr/local/.ghcup\n sudo rm -rf /usr/share/swift\n\n - name: Check out the repo\n uses: actions/checkout@v3\n\n - name: Add a dummy ext/ directory\n run:\n mkdir ext && touch ext/dummy\n\n - name: Check out the ext/ directory\n if: matrix.image.name != 'grist-oss'\n run: buildtools/checkout-ext-directory.sh ${{ matrix.image.repo }}\n\n - name: Generate metadata tag input\n id: meta_input\n run: |\n {\n echo \"tags<> $GITHUB_OUTPUT\n\n - name: Docker meta\n id: meta\n uses: docker/metadata-action@v4\n with:\n images: |\n ${{ env.DOCKER_HUB_OWNER }}/${{ matrix.image.name }}\n tags: |\n ${{ steps.meta_input.outputs.tags }}\n\n - name: Docker meta (EE)\n if: ${{ matrix.image.name == 'grist' }}\n id: meta_ee\n uses: docker/metadata-action@v4\n with:\n images: |\n ${{ env.DOCKER_HUB_OWNER }}/grist-ee\n tags: |\n ${{ steps.meta_input.outputs.tags }}\n\n - name: Set up QEMU\n uses: docker/setup-qemu-action@v1\n\n - name: Set up Docker Buildx\n uses: docker/setup-buildx-action@v1\n\n - name: Log in to Docker Hub\n uses: docker/login-action@v3\n with:\n username: ${{ secrets.DOCKER_USERNAME }}\n password: ${{ secrets.DOCKER_PASSWORD }}\n\n - name: Push to Docker Hub\n uses: docker/build-push-action@v2\n with:\n context: .\n build-args: GRIST_ALLOW_AUTOMATIC_VERSION_CHECKING=${{ matrix.image.name == 'grist-oss' && 'false' || 'true' }}\n push: true\n platforms: ${{ env.PLATFORMS }}\n tags: ${{ steps.meta.outputs.tags }}\n cache-from: type=gha\n cache-to: type=gha,mode=max\n build-contexts: ext=ext\n\n - name: Push Enterprise to Docker Hub\n if: ${{ matrix.image.name == 'grist' }}\n uses: docker/build-push-action@v2\n with:\n context: .\n build-args: |\n BASE_IMAGE=${{ env.DOCKER_HUB_OWNER }}/${{ matrix.image.name}}\n BASE_VERSION=${{ env.TAG }}\n GRIST_ALLOW_AUTOMATIC_VERSION_CHECKING=true\n file: ext/Dockerfile\n platforms: ${{ env.PLATFORMS }}\n push: true\n tags: ${{ steps.meta_ee.outputs.tags }}\n cache-from: type=gha\n cache-to: type=gha,mode=max\n" }, { "path": ".github/workflows/docker_latest.yml", "content": "name: Push latest Docker image\n\non:\n push:\n # Trigger if latest_candidate updates. This is automatically done by another\n # workflow whenever tests pass on main - but events don't chain without using\n # personal access tokens so we just use a cron job.\n branches: [ latest_candidate ]\n schedule:\n # Run at 5:41 UTC daily\n - cron: '41 5 * * *'\n workflow_dispatch:\n inputs:\n branch:\n description: \"Branch from which to create the latest Docker image (default: latest_candidate)\"\n type: string\n required: true\n default: latest_candidate\n disable_tests:\n description: \"Should the tests be skipped?\"\n type: boolean\n required: True\n default: False\n platforms:\n description: \"Platforms to build\"\n type: choice\n required: True\n options:\n - linux/amd64\n - linux/arm64/v8\n - linux/amd64,linux/arm64/v8\n default: linux/amd64,linux/arm64/v8\n tag:\n description: \"Tag for the resulting images\"\n type: string\n required: True\n default: 'latest'\n\nenv:\n BRANCH: ${{ inputs.branch || 'latest_candidate' }}\n PLATFORMS: ${{ inputs.platforms || 'linux/amd64,linux/arm64/v8' }}\n TAG: ${{ inputs.tag || 'latest' }}\n DOCKER_HUB_OWNER: ${{ vars.DOCKER_HUB_OWNER || github.repository_owner }}\n\njobs:\n push_to_registry:\n name: Push latest Docker image to Docker Hub\n runs-on: ubuntu-22.04\n if: ${{ vars.RUN_DAILY_BUILD }}\n strategy:\n matrix:\n python-version: [3.11]\n node-version: [22.x]\n image:\n # We build two images, `grist-oss` and `grist`.\n # See https://github.com/gristlabs/grist-core?tab=readme-ov-file#available-docker-images\n - name: \"grist-oss\"\n repo: \"grist-core\"\n - name: \"grist\"\n repo: \"grist-ee\"\n steps:\n - name: Build settings\n run: |\n echo \"Branch: $BRANCH\"\n echo \"Platforms: $PLATFORMS\"\n echo \"Docker Hub Owner: $DOCKER_HUB_OWNER\"\n echo \"Tag: $TAG\"\n\n - name: Free disk space\n run: |\n echo \"Disk space before cleanup:\"\n df -h /\n echo \"Removing Android SDK...\"\n sudo rm -rf /usr/local/lib/android\n df -h /\n echo \"Removing .NET...\"\n sudo rm -rf /usr/share/dotnet\n df -h /\n echo \"Removing Haskell...\"\n sudo rm -rf /usr/local/.ghcup\n df -h /\n echo \"Removing Swift...\"\n sudo rm -rf /usr/share/swift\n df -h /\n echo \"Final disk space:\"\n df -h /\n\n - name: Check out the repo\n uses: actions/checkout@v4\n with:\n ref: ${{ env.BRANCH }}\n\n - name: Add a dummy ext/ directory\n run:\n mkdir ext && touch ext/dummy\n\n - name: Check out the ext/ directory\n if: matrix.image.name != 'grist-oss'\n run: buildtools/checkout-ext-directory.sh ${{ matrix.image.repo }}\n\n - name: Set up QEMU\n uses: docker/setup-qemu-action@v1\n\n - name: Set up Docker Buildx\n uses: docker/setup-buildx-action@v1\n\n - name: Prepare image but do not push it yet\n uses: docker/build-push-action@v2\n with:\n context: .\n load: true\n tags: ${{ env.DOCKER_HUB_OWNER }}/${{ matrix.image.name }}:${{ env.TAG }}\n cache-from: type=gha\n build-contexts: ext=ext\n\n - name: Use Node.js ${{ matrix.node-version }} for testing\n if: ${{ !inputs.disable_tests }}\n uses: actions/setup-node@v1\n with:\n node-version: ${{ matrix.node-version }}\n\n - name: Set up Python ${{ matrix.python-version }} for testing - maybe not needed\n if: ${{ !inputs.disable_tests }}\n uses: actions/setup-python@v2\n with:\n python-version: ${{ matrix.python-version }}\n\n - name: Install Python packages\n if: ${{ !inputs.disable_tests }}\n run: |\n pip install virtualenv\n yarn run install:python\n\n - name: Install Node.js packages\n if: ${{ !inputs.disable_tests }}\n run: yarn install\n\n - name: Disable the ext/ directory\n if: ${{ !inputs.disable_tests }}\n run: mv ext/ ext-disabled/\n\n - name: Build Node.js code\n if: ${{ !inputs.disable_tests }}\n run: yarn run build\n\n - name: Install Google Chrome and chromedriver\n run: buildtools/install_chrome_for_tests.sh -y\n\n - name: Run tests with default settings\n if: ${{ !inputs.disable_tests }}\n run: |\n export TEST_IMAGE=${{ env.DOCKER_HUB_OWNER }}/${{ matrix.image.name }}:${{ env.TAG }}\n export VERBOSE=1\n export DEBUG=1\n export MOCHA_WEBDRIVER_HEADLESS=1\n yarn run test:docker\n\n - name: Run some tests with gvisor and python\n if: ${{ !inputs.disable_tests }}\n run: |\n export TEST_IMAGE=${{ env.DOCKER_HUB_OWNER }}/${{ matrix.image.name }}:${{ env.TAG }}\n export VERBOSE=1\n export DEBUG=1\n export MOCHA_WEBDRIVER_HEADLESS=1\n export GREP_TESTS='should support basic editing'\n export TEST_DOCKER_OPTIONS='-e GRIST_SANDBOX_FLAVOR=gvisor -e PYTHON_VERSION_ON_CREATION=3'\n yarn run test:docker\n\n - name: Re-enable the ext/ directory\n if: ${{ !inputs.disable_tests }}\n run: mv ext-disabled/ ext/\n\n - name: Log in to Docker Hub\n uses: docker/login-action@v1 \n with:\n username: ${{ secrets.DOCKER_USERNAME }}\n password: ${{ secrets.DOCKER_PASSWORD }}\n\n - name: Push to Docker Hub\n uses: docker/build-push-action@v2\n with:\n context: .\n build-args: GRIST_ALLOW_AUTOMATIC_VERSION_CHECKING=${{ matrix.image.name == 'grist-oss' && 'false' || 'true' }}\n platforms: ${{ env.PLATFORMS }}\n push: true\n tags: ${{ env.DOCKER_HUB_OWNER }}/${{ matrix.image.name }}:${{ env.TAG }}\n cache-from: type=gha\n cache-to: type=gha,mode=max\n build-contexts: ext=ext\n\n - name: Push Enterprise to Docker Hub\n if: ${{ matrix.image.name == 'grist' }}\n uses: docker/build-push-action@v2\n with:\n context: .\n build-args: |\n BASE_IMAGE=${{ env.DOCKER_HUB_OWNER }}/${{ matrix.image.name}}\n BASE_VERSION=${{ env.TAG }}\n GRIST_ALLOW_AUTOMATIC_VERSION_CHECKING=true\n file: ext/Dockerfile\n platforms: ${{ env.PLATFORMS }}\n push: true\n tags: ${{ env.DOCKER_HUB_OWNER }}/grist-ee:${{ env.TAG }}\n cache-from: type=gha\n cache-to: type=gha,mode=max\n\n update_latest_branch:\n name: Update latest branch\n runs-on: ubuntu-22.04\n needs: push_to_registry\n steps:\n - name: Check out the repo\n uses: actions/checkout@v2\n with:\n ref: ${{ inputs.latest_branch }}\n\n - name: Update latest branch\n uses: ad-m/github-push-action@8407731efefc0d8f72af254c74276b7a90be36e1\n with:\n github_token: ${{ secrets.GITHUB_TOKEN }}\n branch: latest\n force: true\n" }, { "path": ".github/workflows/fly-build.yml", "content": "# fly-deploy will be triggered on completion of this workflow to actually deploy the code to fly.io.\n\nname: fly.io Build\non:\n pull_request:\n branches: [ main ]\n types: [labeled, opened, synchronize, reopened]\n\n # Allows running this workflow manually from the Actions tab\n workflow_dispatch:\n\njobs:\n build:\n name: Build Docker image\n runs-on: ubuntu-22.04\n # Build when the 'preview' label is added, or when PR is updated with this label present.\n if: >\n github.event_name == 'workflow_dispatch' ||\n (github.event_name == 'pull_request' &&\n contains(github.event.pull_request.labels.*.name, 'preview'))\n steps:\n - uses: actions/checkout@v4\n - name: Build and export Docker image\n id: docker-build\n run: >\n ./buildtools/checkout-ext-directory.sh grist-ee &&\n docker build -t grist-core:preview . --build-context ext=ext &&\n docker image save grist-core:preview -o grist-core.tar\n - name: Save PR information\n run: |\n echo PR_NUMBER=${{ github.event.number }} >> ./pr-info.txt\n echo PR_SOURCE=${{ github.event.pull_request.head.repo.full_name }}-${{ github.event.pull_request.head.ref }} >> ./pr-info.txt\n echo PR_SHASUM=${{ github.event.pull_request.head.sha }} >> ./pr-info.txt\n # PR_SOURCE looks like /-.\n # For example, if the GitHub user \"foo\" forked grist-core as \"grist-bar\", and makes a PR from their branch named \"baz\",\n # it will be \"foo/grist-bar-baz\". deploy.js later replaces \"/\" with \"-\", making it \"foo-grist-bar-baz\".\n - name: Upload artifact\n uses: actions/upload-artifact@v4\n with:\n name: docker-image\n path: |\n ./grist-core.tar\n ./pr-info.txt\n ./buildtools/fly-template.env\n if-no-files-found: \"error\"\n" }, { "path": ".github/workflows/fly-cleanup.yml", "content": "name: fly.io Cleanup\non:\n schedule:\n # Once a day, clean up jobs marked as expired\n - cron: '50 12 * * *'\n\n # Allows running this workflow manually from the Actions tab\n workflow_dispatch:\n\nenv:\n FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}\n\njobs:\n clean:\n name: Clean stale deployed apps\n runs-on: ubuntu-22.04\n if: github.repository_owner == 'gristlabs'\n steps:\n - uses: actions/checkout@v3\n - uses: superfly/flyctl-actions/setup-flyctl@master\n with:\n version: 0.2.72\n - run: node buildtools/fly-deploy.js clean\n" }, { "path": ".github/workflows/fly-deploy.yml", "content": "# Follow-up of fly-build, with access to secrets for making deployments.\n# This workflow runs in the target repo context. It does not, and should never execute user-supplied code.\n# See https://securitylab.github.com/research/github-actions-preventing-pwn-requests/\n\nname: fly.io Deploy\non:\n workflow_run:\n workflows: [\"fly.io Build\"]\n types:\n - completed\n\njobs:\n deploy:\n name: Deploy app to fly.io\n runs-on: ubuntu-22.04\n if: |\n github.event.workflow_run.event == 'pull_request' &&\n github.event.workflow_run.conclusion == 'success'\n steps:\n - uses: actions/checkout@v4\n - name: Set up flyctl\n uses: superfly/flyctl-actions/setup-flyctl@master\n with:\n version: 0.2.72\n - name: Download artifacts\n uses: actions/github-script@v7\n with:\n script: |\n var artifacts = await github.rest.actions.listWorkflowRunArtifacts({\n owner: context.repo.owner,\n repo: context.repo.repo,\n run_id: ${{ github.event.workflow_run.id }},\n });\n var matchArtifact = artifacts.data.artifacts.filter((artifact) => {\n return artifact.name == \"docker-image\"\n })[0];\n var download = await github.rest.actions.downloadArtifact({\n owner: context.repo.owner,\n repo: context.repo.repo,\n artifact_id: matchArtifact.id,\n archive_format: 'zip',\n });\n var fs = require('fs');\n fs.writeFileSync('${{github.workspace}}/docker-image.zip', Buffer.from(download.data));\n await github.rest.actions.deleteArtifact({\n owner: context.repo.owner,\n repo: context.repo.repo,\n artifact_id: matchArtifact.id,\n });\n - name: Extract artifacts\n id: extract_artifacts\n run: |\n unzip -o docker-image.zip grist-core.tar pr-info.txt buildtools/fly-template.env\n cat ./pr-info.txt >> $GITHUB_OUTPUT\n - name: Load Docker image\n run: docker load --input grist-core.tar\n - name: Deploy to fly.io\n id: fly_deploy\n env:\n FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}\n BRANCH_NAME: ${{ steps.extract_artifacts.outputs.PR_SOURCE }}\n run: |\n node buildtools/fly-deploy.js deploy\n flyctl config -c ./fly.toml env | awk '/APP_HOME_URL/{print \"DEPLOY_URL=\" $2}' >> $GITHUB_OUTPUT\n flyctl config -c ./fly.toml env | awk '/FLY_DEPLOY_EXPIRATION/{print \"EXPIRES=\" $2}' >> $GITHUB_OUTPUT\n - name: Comment on PR\n uses: actions/github-script@v7\n with:\n script: |\n github.rest.issues.createComment({\n issue_number: ${{ steps.extract_artifacts.outputs.PR_NUMBER }},\n owner: context.repo.owner,\n repo: context.repo.repo,\n body: `Deployed commit \\`${{ steps.extract_artifacts.outputs.PR_SHASUM }}\\` as ${{ steps.fly_deploy.outputs.DEPLOY_URL }} (until ${{ steps.fly_deploy.outputs.EXPIRES }})`\n })\n" }, { "path": ".github/workflows/fly-destroy.yml", "content": "# This workflow runs in the target repo context, as it is triggered via pull_request_target.\n# It does not, and should not have access to code in the PR.\n# See https://securitylab.github.com/research/github-actions-preventing-pwn-requests/\n\nname: fly.io Destroy\non:\n pull_request_target:\n branches: [ main ]\n types: [unlabeled, closed]\n\n # Allows running this workflow manually from the Actions tab\n workflow_dispatch:\n\njobs:\n destroy:\n name: Remove app from fly.io\n runs-on: ubuntu-22.04\n # Remove the deployment when 'preview' label is removed, or the PR is closed.\n if: |\n github.event_name == 'workflow_dispatch' ||\n (github.event_name == 'pull_request_target' &&\n (github.event.action == 'closed' ||\n (github.event.action == 'unlabeled' && github.event.label.name == 'preview')))\n steps:\n - uses: actions/checkout@v4\n - name: Set up flyctl\n uses: superfly/flyctl-actions/setup-flyctl@master\n with:\n version: 0.2.72\n - name: Destroy fly.io app\n env:\n FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}\n BRANCH_NAME: ${{ github.event.pull_request.head.repo.full_name }}-${{ github.event.pull_request.head.ref }}\n # See fly-build for what BRANCH_NAME looks like.\n id: fly_destroy\n run: node buildtools/fly-deploy.js destroy\n" }, { "path": ".github/workflows/main.yml", "content": "name: CI\n\non:\n push:\n branches: [ main ]\n pull_request:\n branches: [ main ]\n\n # Allows running this workflow manually from the Actions tab\n workflow_dispatch:\n\njobs:\n build_and_test:\n runs-on: ${{ matrix.os }}\n strategy:\n # it is helpful to know which sets of tests would have succeeded,\n # even when there is a failure.\n fail-fast: false\n matrix:\n os: ['ubuntu-24.04']\n python-version: [3.11]\n node-version: [22.x]\n tests:\n - ':lint:python:client:common:smoke:stubs:pyodide:'\n - ':server-1-of-2:'\n - ':server-2-of-2:'\n - ':gen-server:'\n - ':nbrowser-^[A-D]:'\n - ':nbrowser-^[E-L]:'\n - ':nbrowser-^[M-N]:'\n - ':nbrowser-^[O-R]:'\n - ':nbrowser-^[^A-R]:'\n - ':projects:'\n include:\n - tests: ':lint:python:client:common:smoke:'\n node-version: 22.x\n python-version: '3.10'\n os: ubuntu-24.04\n - tests: ':pyodide:macsandbox:'\n node-version: 22.x\n python-version: '3.11'\n os: macos-latest\n steps:\n - uses: actions/checkout@v3\n\n - name: Use Node.js ${{ matrix.node-version }}\n uses: actions/setup-node@v3\n with:\n node-version: ${{ matrix.node-version }}\n cache: 'yarn'\n\n - name: Set up Python ${{ matrix.python-version }}\n uses: actions/setup-python@v4\n with:\n python-version: ${{ matrix.python-version }}\n cache: 'pip'\n\n - name: Install Python packages\n run: |\n pip install virtualenv\n yarn run install:python\n\n - name: Install Node.js packages\n run: yarn install\n\n - name: Install gvisor\n if: contains(matrix.os, 'ubuntu')\n run: |\n docker create --name temp-runsc gristlabs/gvisor-unprivileged:buster /bin/true\n sudo docker cp temp-runsc:/runsc /usr/bin/runsc\n docker rm temp-runsc\n\n - name: Run eslint\n if: contains(matrix.tests, ':lint:')\n run: yarn run lint:ci\n\n - name: Make sure bucket is versioned\n if: contains(matrix.os, 'ubuntu') && contains(matrix.tests, ':server-') || contains(matrix.os, 'ubuntu') && contains(matrix.tests, ':gen-server:')\n env:\n AWS_ACCESS_KEY_ID: administrator\n AWS_SECRET_ACCESS_KEY: administrator\n run: aws --region us-east-1 --endpoint-url http://localhost:9000 s3api put-bucket-versioning --bucket grist-docs-test --versioning-configuration Status=Enabled\n\n - name: Build Node.js code\n run: yarn run build\n\n - name: Install Google Chrome and chromedriver\n if: contains(matrix.tests, ':nbrowser-') || contains(matrix.tests, ':smoke:') || contains(matrix.tests, ':stubs:') || contains(matrix.tests, ':projects:')\n run: buildtools/install_chrome_for_tests.sh -y\n\n - name: Run smoke test\n if: contains(matrix.tests, ':smoke:')\n run: VERBOSE=1 DEBUG=1 MOCHA_WEBDRIVER_HEADLESS=1 yarn run test:smoke\n\n - name: Run python tests\n if: contains(matrix.tests, ':python:')\n run: yarn run test:python\n\n - name: Run client tests\n if: contains(matrix.tests, ':client:')\n run: yarn run test:client\n\n - name: Run common tests\n if: contains(matrix.tests, ':common:')\n run: yarn run test:common\n\n - name: Run stubs tests\n if: contains(matrix.tests, ':stubs:')\n run: MOCHA_WEBDRIVER_HEADLESS=1 yarn run test:stubs\n\n - name: Run gen-server tests with sqlite, minio and redis\n if: contains(matrix.tests, ':gen-server:')\n run: |\n yarn run test:gen-server\n # Anchors should be used once available. Not supported yet as of December 2024.\n # https://github.com/actions/runner/issues/1182\n env:\n MOCHA_WEBDRIVER_HEADLESS: 1\n TESTS: ${{ matrix.tests }}\n GRIST_DOCS_MINIO_ACCESS_KEY: administrator\n GRIST_DOCS_MINIO_SECRET_KEY: administrator\n TEST_REDIS_URL: \"redis://localhost/11\"\n GRIST_DOCS_MINIO_USE_SSL: 0\n GRIST_DOCS_MINIO_ENDPOINT: localhost\n GRIST_DOCS_MINIO_PORT: 9000\n GRIST_DOCS_MINIO_BUCKET: grist-docs-test\n\n - name: Run a couple of tests using pyodide\n if: contains(matrix.tests, ':pyodide:')\n run: |\n cd sandbox/pyodide\n make setup\n cd ../..\n yarn run test:server -g 'ActiveDoc.useQuerySet|Sandbox'\n yarn run test:nbrowser -g 'Importer.*should.show.correct.preview'\n env:\n MOCHA_WEBDRIVER_HEADLESS: 1\n GRIST_SANDBOX_FLAVOR: pyodide\n\n - name: Run a couple of tests using macSandboxExec\n if: contains(matrix.tests, ':macsandbox:')\n run: |\n yarn run test:server -g Sandbox\n env:\n MOCHA_WEBDRIVER_HEADLESS: 1\n GRIST_SANDBOX_FLAVOR: macSandboxExec\n\n - name: Run gen-server tests with postgres, minio and redis\n if: contains(matrix.tests, ':gen-server:')\n run: |\n PGPASSWORD=$TYPEORM_PASSWORD psql -h $TYPEORM_HOST -U $TYPEORM_USERNAME -w $TYPEORM_DATABASE -c \"SHOW ALL;\" | grep ' jit '\n yarn run test:gen-server\n env:\n MOCHA_WEBDRIVER_HEADLESS: 1\n TESTS: ${{ matrix.tests }}\n GRIST_DOCS_MINIO_ACCESS_KEY: administrator\n GRIST_DOCS_MINIO_SECRET_KEY: administrator\n TEST_REDIS_URL: \"redis://localhost/11\"\n GRIST_DOCS_MINIO_USE_SSL: 0\n GRIST_DOCS_MINIO_ENDPOINT: localhost\n GRIST_DOCS_MINIO_PORT: 9000\n GRIST_DOCS_MINIO_BUCKET: grist-docs-test\n TYPEORM_TYPE: postgres\n TYPEORM_HOST: localhost\n TYPEORM_DATABASE: db_name\n TYPEORM_USERNAME: db_user\n TYPEORM_PASSWORD: db_password\n\n - name: Run server tests with minio and redis\n if: contains(matrix.tests, ':server-')\n run: |\n export TEST_SPLITS=$(echo $TESTS | sed \"s/.*:server-\\([^:]*\\).*/\\1/\")\n yarn run test:server\n env:\n MOCHA_WEBDRIVER_HEADLESS: 1\n TESTS: ${{ matrix.tests }}\n GRIST_DOCS_MINIO_ACCESS_KEY: administrator\n GRIST_DOCS_MINIO_SECRET_KEY: administrator\n TEST_REDIS_URL: \"redis://localhost/11\"\n GVISOR_FLAGS: \"-unprivileged -ignore-cgroups\"\n GVISOR_EXTRA_DIRS: /opt\n GRIST_DOCS_MINIO_USE_SSL: 0\n GRIST_DOCS_MINIO_ENDPOINT: localhost\n GRIST_DOCS_MINIO_PORT: 9000\n GRIST_DOCS_MINIO_BUCKET: grist-docs-test\n\n - name: Run main tests without minio and redis\n if: contains(matrix.tests, ':nbrowser-')\n run: |\n mkdir -p $MOCHA_WEBDRIVER_LOGDIR\n export GREP_TESTS=$(echo $TESTS | sed \"s/.*:nbrowser-\\([^:]*\\).*/\\1/\")\n MOCHA_WEBDRIVER_SKIP_CLEANUP=1 MOCHA_WEBDRIVER_HEADLESS=1 yarn run test:nbrowser --parallel --jobs 3\n env:\n TESTS: ${{ matrix.tests }}\n MOCHA_WEBDRIVER_LOGDIR: ${{ runner.temp }}/test-logs/webdriver\n GVISOR_FLAGS: \"-unprivileged -ignore-cgroups\"\n GVISOR_EXTRA_DIRS: /opt\n TESTDIR: ${{ runner.temp }}/test-logs\n\n - name: Run projects tests\n if: contains(matrix.tests, ':projects:')\n run: |\n mkdir -p $MOCHA_WEBDRIVER_LOGDIR\n yarn run test:projects\n env:\n MOCHA_WEBDRIVER_LOGDIR: ${{ runner.temp }}/test-logs/webdriver\n TESTDIR: ${{ runner.temp }}/test-logs\n MOCHA_WEBDRIVER_HEADLESS: 1\n\n - name: Prepare for saving artifact\n if: failure()\n run: |\n ARTIFACT_NAME=logs-$(echo $TESTS | sed 's/[^-a-zA-Z0-9]/_/g')\n echo \"Artifact name is '$ARTIFACT_NAME'\"\n echo \"ARTIFACT_NAME=$ARTIFACT_NAME\" >> $GITHUB_ENV\n mkdir -p $TESTDIR\n find $TESTDIR -iname \"*.socket\" -exec rm {} \\;\n env:\n TESTS: ${{ matrix.tests }}\n TESTDIR: ${{ runner.temp }}/test-logs\n\n - name: Save artifacts on failure\n if: failure()\n uses: actions/upload-artifact@v4\n with:\n name: ${{ env.ARTIFACT_NAME }}\n path: ${{ runner.temp }}/test-logs # only exists for webdriver tests\n\n services:\n # https://github.com/bitnami/containers/issues/83267\n minio:\n image: ${{ matrix.os == 'ubuntu-24.04' && 'bitnamilegacy/minio:2025.4.22' || '' }} \n env:\n MINIO_DEFAULT_BUCKETS: \"grist-docs-test:public\"\n MINIO_ROOT_USER: administrator\n MINIO_ROOT_PASSWORD: administrator\n ports:\n - 9000:9000\n options: >-\n --health-cmd \"curl -f http://localhost:9000/minio/health/ready\"\n --health-interval 10s\n --health-timeout 5s\n --health-retries 5\n\n redis:\n image: ${{ matrix.os == 'ubuntu-24.04' && 'redis' || '' }}\n ports:\n - 6379:6379\n options: >-\n --health-cmd \"redis-cli ping\"\n --health-interval 10s\n --health-timeout 5s\n --health-retries 5\n\n postgresql:\n image: ${{ matrix.os == 'ubuntu-24.04' && 'postgres:latest' || '' }}\n env:\n POSTGRES_USER: db_user\n POSTGRES_PASSWORD: db_password\n POSTGRES_DB: db_name\n # JIT is enabled by default since Postgres 17 and has a huge negative impact on performance,\n # making many tests timeout.\n # https://support.getgrist.com/self-managed/#what-is-a-home-database\n POSTGRES_INITDB_ARGS: \"-c jit=off\"\n ports:\n - 5432:5432\n options: >-\n --health-cmd \"pg_isready -U db_user\"\n --health-interval 10s\n --health-timeout 5s\n --health-retries 5\n\n candidate:\n needs: build_and_test\n if: ${{ success() && github.event_name == 'push' }}\n runs-on: ubuntu-22.04\n steps:\n - name: Fetch new candidate branch\n uses: actions/checkout@v3\n\n - name: Update candidate branch\n uses: ad-m/github-push-action@8407731efefc0d8f72af254c74276b7a90be36e1\n with:\n github_token: ${{ secrets.GITHUB_TOKEN }}\n branch: latest_candidate\n force: true\n" }, { "path": ".github/workflows/self-hosted.yml", "content": "name: Add self-hosting issues to the self-hosting project\n\non:\n issues:\n types:\n - opened\n - labeled\n\njobs:\n add-to-project:\n name: Add issue to project\n runs-on: ubuntu-22.04\n steps:\n - uses: actions/add-to-project@v1.0.1\n with:\n project-url: https://github.com/orgs/gristlabs/projects/2\n github-token: ${{ secrets.SELF_HOSTED_PROJECT }}\n labeled: self-hosting\n" }, { "path": ".github/workflows/translation_keys.yml", "content": "name: Translation keys\n\non:\n push:\n branches: [ main ]\n workflow_dispatch:\n\npermissions:\n pull-requests: write\n contents: write\n\njobs:\n build:\n if: github.repository_owner == 'gristlabs'\n runs-on: ubuntu-22.04\n steps:\n - uses: actions/checkout@v2\n with:\n fetch-depth: 0 # Let's get all the branches\n\n - name: Use Node.js\n uses: actions/setup-node@v1\n with:\n node-version: 22\n\n - name: Install Node.js packages\n run: yarn install\n\n - name: Build code\n run: yarn run build\n\n - name: Scan for keys\n id: scan-keys\n run: |\n git checkout -b translation-keys\n yarn run generate:translation 2>&1 | tee /tmp/scan-output.txt\n git status --porcelain\n if [[ $(git status --porcelain | wc -l) -eq \"0\" ]]; then\n echo \"No changes\"\n echo \"CHANGED=false\" >> $GITHUB_ENV\n else\n echo \"Changes detected\"\n echo \"CHANGED=true\" >> $GITHUB_ENV\n fi\n\n - name: setup git config\n run: |\n git config user.name \"Paul's Grist Bot\"\n git config user.email \"\"\n\n - name: Create PR and merge\n if: env.CHANGED == 'true'\n run: |\n git commit -m \"automated update to translation keys\" -a\n git push --set-upstream origin HEAD:translation-keys -f\n num=$(gh pr list --search \"automated update to translation keys\" --json number -q \".[].number\")\n if [[ \"$num\" != \"\" ]]; then\n echo \"Existing translation keys PR #$num is open, skipping\"\n exit 0\n fi\n sed -n '/TRANSLATION_SUMMARY_START/,/TRANSLATION_SUMMARY_END/{//d;p;}' /tmp/scan-output.txt > /tmp/pr-body.txt\n gh pr create --title \"automated update to translation keys\" --body-file /tmp/pr-body.txt\n gh pr merge --merge --delete-branch\n env:\n GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n" }, { "path": ".gitignore", "content": "/node_modules/\n/_build/\n/static/*.bundle.js\n/static/*.bundle.js.map\n/static/grist-plugin-api*\n/static/bundle.css\n/static/browser-check.js\n/static/*.bundle.js.*.txt\n/grist-sessions.db\n/landing.db\n/docs/\n/sandbox_venv*\n/.vscode/\n\n# Files created by grist-desktop setup\n/cpython.tar.gz\n/python\n/static_ext\n\n# Build helper files.\n/.build*\n\n*.swp\n*.pyc\n*.bak\n.DS_Store\n\n# Logs\nlogs\n*.log\nnpm-debug.log*\nyarn-debug.log*\nyarn-error.log*\n\n# Runtime data\npids\n*.pid\n*.seed\n*.pid.lock\n\n# Directory for instrumented libs generated by jscoverage/JSCover\nlib-cov\n\n# Coverage directory used by tools like istanbul\ncoverage\n\n# nyc test coverage\n.nyc_output\n\n# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)\n.grunt\n\n# node-waf configuration\n.lock-wscript\n\n# Compiled binary addons (http://nodejs.org/api/addons.html)\nbuild/Release\n\n# Dependency directories\n/node_modules/\njspm_packages/\n/docs\n\n# Optional npm cache directory\n.npm\n\n# Optional eslint cache\n.eslintcache\n\n# Optional REPL history\n.node_repl_history\n\n# Output of 'npm pack'\n*.tgz\n\n# Yarn Integrity file\n.yarn-integrity\n\n# dotenv environment variables file\n.env\n\n# Test\ntimings.txt\nxunit.xml\n.clipboard.lock\n\n**/_build\n\n# ext directory can be overwritten\n/ext\n/ext/**\n\n# Docker compose examples - persistent values and secrets\n/docker-compose-examples/*/persist\n/docker-compose-examples/*/secrets\n/docker-compose-examples/grist-traefik-oidc-auth/.env\n\n# Sample grist documents\n/samples/\n\n/test/assistant/data/cache/\n/test/assistant/data/results/\n/test/assistant/data/templates/\n" }, { "path": ".nvmrc", "content": "v22.12.0\n" }, { "path": ".yarnrc", "content": "# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.\n# yarn lockfile v1\n\n\nyarn-offline-mirror false\n" }, { "path": "CONTRIBUTING.md", "content": "# Welcome to the contribution guide for Grist!\n\nYou are eager to contribute to Grist? That's awesome! See below some contributions you can make:\n- [translate](/documentation/translations.md)\n- [write tutorials and user documentation](https://github.com/gristlabs/grist-help?tab=readme-ov-file#grist-help-center)\n- [develop](/documentation/develop.md)\n- [report issues or suggest enhancement](https://github.com/gristlabs/grist-core/issues/new/choose)\n\n" }, { "path": "Dockerfile", "content": "################################################################################\n## The Grist source can be extended. This is a stub that can be overridden\n## from command line, as:\n## docker buildx build -t ... --build-context=ext= .\n## The code in will then be built along with the rest of Grist.\n################################################################################\nFROM scratch AS ext\n\n################################################################################\n## Javascript build stage\n################################################################################\n\nFROM node:22-trixie AS prod-builder\n\n# Install all node dependencies.\nWORKDIR /grist\nCOPY package.json yarn.lock /grist/\nRUN \\\n yarn install --prod --frozen-lockfile --verbose --network-timeout 600000\n\nFROM prod-builder AS builder\n\n# Create node_modules with devDependencies to be able to build the app\n# Add at global level gyp deps to build sqlite3 for prod\n# then create node_modules_prod that will be the node_modules of final image\nRUN \\\n yarn install --frozen-lockfile --verbose --network-timeout 600000\n\n# Install any extra node dependencies (at root level, to avoid having to wrestle\n# with merging them).\nCOPY --from=ext / /grist/ext\nRUN \\\n mkdir /node_modules && \\\n cd /grist/ext && \\\n { if [ -e package.json ] ; then yarn install --frozen-lockfile --modules-folder=/node_modules --verbose --network-timeout 600000 ; fi }\n\n# Build node code.\nCOPY tsconfig.json /grist\nCOPY tsconfig-ext.json /grist\nCOPY tsconfig-prod.json /grist\nCOPY test/tsconfig.json /grist/test/tsconfig.json\nCOPY test/chai-as-promised.js /grist/test/chai-as-promised.js\nCOPY app /grist/app\nCOPY stubs /grist/stubs\nCOPY buildtools /grist/buildtools\n# Copy locales files early. During build process they are validated.\nCOPY static/locales /grist/static/locales\nRUN WEBPACK_EXTRA_MODULE_PATHS=/node_modules yarn run build:prod\n# We don't need them anymore, they will by copied to the final image.\nRUN rm -rf /grist/static/locales\n\n\n# Prepare material for optional pyodide sandbox\nCOPY sandbox/pyodide /grist/sandbox/pyodide\nCOPY sandbox/requirements.txt /grist/sandbox/requirements.txt\nRUN \\\n cd /grist/sandbox/pyodide && make setup\n\n################################################################################\n## Python collection stage\n################################################################################\n\n# Fetch python3.11\nFROM python:3.11-slim-trixie AS collector-py3\nCOPY sandbox/requirements.txt requirements.txt\nRUN \\\n pip3 install -r requirements.txt\n\n################################################################################\n## Sandbox collection stage\n################################################################################\n\n# Fetch gvisor-based sandbox. Note, to enable it to run within default\n# unprivileged docker, layers of protection that require privilege have\n# been stripped away, see https://github.com/google/gvisor/issues/4371\n# The standalone sandbox binary is built on buster, but remains compatible\n# with recent Debian.\n# If you'd like to use unmodified gvisor, you should be able to just drop\n# in the standard runsc binary and run the container with any extra permissions\n# it needs.\nFROM docker.io/gristlabs/gvisor-unprivileged:buster AS sandbox\n\n################################################################################\n## Run-time stage\n################################################################################\n\n# Now, start preparing final image.\nFROM node:22-trixie-slim\n\nARG GRIST_ALLOW_AUTOMATIC_VERSION_CHECKING=false\n\n# Install curl for docker healthchecks, libexpat1 and libsqlite3-0 for python3\n# library binary dependencies, and procps for managing gvisor processes.\nRUN \\\n apt-get update && \\\n apt-get install -y --no-install-recommends curl libexpat1 libsqlite3-0 procps tini && \\\n rm -rf /var/lib/apt/lists/*\n\n# Keep all storage user may want to persist in a distinct directory\nRUN mkdir -p /persist/docs\n\n# Copy node files.\nCOPY --from=builder /node_modules /node_modules\nCOPY --from=prod-builder /grist/node_modules /grist/node_modules\nCOPY --from=builder /grist/_build /grist/_build\nCOPY --from=builder /grist/static /grist/static-built\nCOPY --from=builder /grist/app/cli.sh /grist/cli\n# Patterm match here is to copy assets only if it exists in the\n# builder stage, otherwise matches nothing.\n# https://stackoverflow.com/a/70096420/11352427\nCOPY --from=builder /grist/ext/asset[s] /grist/ext/assets\n\n# Copy python3 files.\nCOPY --from=collector-py3 /usr/local/bin/python3.11 /usr/bin/python3.11\nCOPY --from=collector-py3 /usr/local/lib/python3.11 /usr/local/lib/python3.11\nCOPY --from=collector-py3 /usr/local/lib/libpython3.11.* /usr/local/lib/\n# Set default to python3\nRUN \\\n ln -s /usr/bin/python3.11 /usr/bin/python && \\\n ln -s /usr/bin/python3.11 /usr/bin/python3 && \\\n ldconfig\n\n# Copy runsc.\nCOPY --from=sandbox /runsc /usr/bin/runsc\n\n# Add files needed for running server.\nCOPY package.json /grist/package.json\nCOPY bower_components /grist/bower_components\nCOPY sandbox /grist/sandbox\nCOPY plugins /grist/plugins\nCOPY static /grist/static\n\n# Make optional pyodide sandbox available\nCOPY --from=builder /grist/sandbox/pyodide /grist/sandbox/pyodide\n\n# Finalize static directory\nRUN \\\n mv /grist/static-built/* /grist/static && \\\n rmdir /grist/static-built\n\n# To ensure non-root users can run grist, 'other' users need read access (and execute on directories)\n# This should be the case by default when copying files in.\n# Only uncomment this if running into permissions issues, as it takes a long time to execute on some systems.\n# RUN chmod -R o+rX /grist\n\n# Add a user to allow de-escalating from root on startup\nRUN useradd -ms /bin/bash grist\nENV GRIST_DOCKER_USER=grist \\\n GRIST_DOCKER_GROUP=grist\nWORKDIR /grist\n\n# Set some default environment variables to give a setup that works out of the box when\n# started as:\n# docker run -p 8484:8484 -it \n# Variables will need to be overridden for other setups.\n#\n# GRIST_SANDBOX_FLAVOR is set to unsandboxed by default, because it\n# appears that the services people use to run docker containers have\n# a wide variety of security settings and the functionality needed for\n# sandboxing may not be possible in every case. For default docker\n# settings, you can get sandboxing as follows:\n# docker run --env GRIST_SANDBOX_FLAVOR=gvisor -p 8484:8484 -it \n#\n# \"NODE_OPTIONS=--no-deprecation\" is set because there is a punycode\n# deprecation nag that is relevant to developers but not to users.\n# TODO: upgrade package.json to avoid using all package versions\n# using the punycode functionality that may be removed in future\n# versions of node.\n#\n# \"NODE_ENV=production\" gives ActiveDoc operations more time to\n# complete, and the express webserver also does some streamlining\n# with this setting. If you don't want these, set NODE_ENV to\n# development.\n#\nENV \\\n GRIST_ORG_IN_PATH=true \\\n GRIST_HOST=0.0.0.0 \\\n GRIST_SINGLE_PORT=true \\\n GRIST_SERVE_SAME_ORIGIN=true \\\n GRIST_DATA_DIR=/persist/docs \\\n GRIST_INST_DIR=/persist \\\n GRIST_SESSION_COOKIE=grist_core \\\n GRIST_ALLOW_AUTOMATIC_VERSION_CHECKING=${GRIST_ALLOW_AUTOMATIC_VERSION_CHECKING} \\\n GVISOR_FLAGS=\"-unprivileged -ignore-cgroups\" \\\n GRIST_SANDBOX_FLAVOR=unsandboxed \\\n NODE_OPTIONS=\"--no-deprecation\" \\\n NODE_ENV=production \\\n TYPEORM_DATABASE=/persist/home.sqlite3\n\nEXPOSE 8484\n\n# When run without any arguments, we run the Grist server within\n# a simple supervisor.\n# When arguments are supplied they are treated as a command to run,\n# as is default for docker. We arrange to have a \"cli\" command that\n# is the same as \"yarn cli\" run from the source code repo.\n# So you can do things like:\n# docker run --rm -v $PWD:$PWD -it gristlabs/grist \\\n# cli sqlite query $PWD/docs/4gtUhAEGbGAdsGNc52k4H6.grist \\\n# --json \"select * from _gristsys_ActionHistory\"\n\nENTRYPOINT [\"./sandbox/docker_entrypoint.sh\"]\nCMD [\"node\", \"./sandbox/supervisor.mjs\"]\n" }, { "path": "LICENSE.txt", "content": "\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright 2014-2022 Grist Labs Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "NOTICE.txt", "content": "Grist Software\nCopyright 2014-2022 Grist Labs Inc.\n\nThis product includes software developed at\nGrist Labs Inc. (https://www.getgrist.com/).\n" }, { "path": "README.md", "content": "# Grist\n\nGrist is a modern relational spreadsheet. It combines the flexibility of a spreadsheet with the robustness of a database.\n\n* `grist-core` (this repo) has what you need to run a powerful server for hosting spreadsheets.\n\n* [`grist-desktop`](https://github.com/gristlabs/grist-desktop) is a Linux/macOS/Windows desktop app for viewing and editing spreadsheets stored locally.\n* [`grist-static`](https://github.com/gristlabs/grist-static) is a fully in-browser build of Grist for displaying spreadsheets on a website without back-end support.\n\nGrist is developed by [Grist Labs](https://www.linkedin.com/company/grist-labs/), an NYC-based company 🇺🇸🗽. The French government 🇫🇷 organizations [ANCT Données et Territoires](https://donnees.incubateur.anct.gouv.fr/toolbox/grist) and [DINUM (Direction Interministérielle du Numérique)](https://www.numerique.gouv.fr/dinum/) have also made significant contributions to the codebase.\n\nThe `grist-core`, `grist-desktop`, and `grist-static` repositories are all open source (Apache License, Version 2.0).\nGrist Labs offers free and paid hosted services at [getgrist.com](https://getgrist.com), sells an Enterprise product,\nand offers [cloud packaging](https://support.getgrist.com/install/grist-builder-edition/).\n\n> Questions? Feedback? Want to share what you're building with Grist? Join our [official Discord server](https://discord.gg/MYKpYQ3fbP) or visit our [Community forum](https://community.getgrist.com/). \n>\n> To keep up-to-date with everything that's going on, you can [sign up for Grist's monthly newsletter](https://www.getgrist.com/newsletter/).\n\nhttps://github.com/user-attachments/assets/fe152f60-3d15-4b11-8cb2-05731a90d273\n\n## Features in `grist-core`\n\nTo see exactly what is present in `grist-core`, you can run the [desktop app](https://github.com/gristlabs/grist-desktop), or use [`docker`](#using-grist). The absolute fastest way to try Grist out is to visit [docs.getgrist.com](https://docs.getgrist.com) and play with a spreadsheet there immediately – though if you do, please read the list of [extra extensions](#features-not-in-grist-core) that are not in `grist-core`.\n\nHowever you try it, you'll quickly see that Grist is a hybrid database/spreadsheet, meaning that:\n\n - Columns work like they do in databases: they are named, and they hold one kind of data.\n - Columns can be filled by formula, spreadsheet-style, with automatic updates when referenced cells change.\n\nThis difference can confuse people coming directly from Excel or Google Sheets. Give it a chance! There's also a [Grist for Spreadsheet Users](https://www.getgrist.com/blog/grist-for-spreadsheet-users/) article to help get you oriented. If you're coming from Airtable, you'll find the model familiar (and there's also our [Grist vs Airtable](https://www.getgrist.com/blog/grist-v-airtable/) article for a direct comparison).\n\nHere are some specific feature highlights of Grist:\n\n * Python formulas.\n - Full [Python syntax is supported](https://support.getgrist.com/formulas/#python), including the standard library.\n - Many [Excel functions](https://support.getgrist.com/functions/) also available.\n - An [AI Assistant](https://www.getgrist.com/ai-formula-assistant/) specifically tuned for formula generation (using OpenAI gpt-3.5-turbo or [Llama](https://ai.meta.com/llama/) via llama-cpp-python).\n * A portable, self-contained format.\n - Based on SQLite, the most widely deployed database engine.\n - Any tool that can read SQLite can read numeric and text data from a Grist file.\n - Enables [backups](https://support.getgrist.com/exports/#backing-up-an-entire-document) that you can confidently restore in full.\n - Great for moving between different hosts.\n * Can be displayed on a static website with [`grist-static`](https://github.com/gristlabs/grist-static) – no special server needed.\n * A self-contained desktop app for viewing and editing locally: [`grist-desktop`](https://github.com/gristlabs/grist-desktop).\n * Convenient editing and formatting features.\n - Choices and [choice lists](https://support.getgrist.com/col-types/#choice-list-columns), for adding colorful tags to records.\n - [References](https://support.getgrist.com/col-refs/#creating-a-new-reference-list-column) and reference lists, for cross-referencing records in other tables.\n - [Attachments](https://support.getgrist.com/col-types/#attachment-columns), to include media or document files in records.\n - Dates and times, toggles, and special numerics such as currency all have specialized editors and formatting options.\n - [Conditional Formatting](https://support.getgrist.com/conditional-formatting/), letting you control the style of cells with formulas to draw attention to important information.\n * Drag-and-drop dashboards.\n - [Charts](https://support.getgrist.com/widget-chart/), [card views](https://support.getgrist.com/widget-card/) and a [calendar widget](https://support.getgrist.com/widget-calendar/) for visualization.\n - [Summary tables](https://support.getgrist.com/summary-tables/) for summing and counting across groups.\n - [Widget linking](https://support.getgrist.com/linking-widgets/) streamlines filtering and editing data.\n Grist has a unique approach to visualization, where you can lay out and link distinct widgets to show together,\n without cramming mixed material into a table.\n - [Filter bar](https://support.getgrist.com/search-sort-filter/#filter-buttons) for quick slicing and dicing.\n * [Incremental imports](https://support.getgrist.com/imports/#updating-existing-records).\n - Import a CSV of the last three months activity from your bank...\n - ...and import new activity a month later without fuss or duplication.\n * [Native forms](https://support.getgrist.com/widget-form/). Create forms that feed directly into your spreadsheet without fuss.\n * Integrations.\n - A [REST API](https://support.getgrist.com/api/), [Zapier actions/triggers](https://support.getgrist.com/integrators/#integrations-via-zapier), and support from similar [integrators](https://support.getgrist.com/integrators/).\n - Import/export to Google drive, Excel format, CSV.\n - Link data with [custom widgets](https://support.getgrist.com/widget-custom/#_top), hosted externally.\n - Configurable outgoing webhooks.\n * [Many templates](https://templates.getgrist.com/) to get you started, from investment research to organizing treasure hunts.\n * Access control options.\n - (You'll need SSO logins set up to make use of these options; [`grist-omnibus`](https://github.com/gristlabs/grist-omnibus) has a prepackaged solution if configuring this feels daunting)\n - Share [individual documents](https://support.getgrist.com/sharing/), workspaces, or [team sites](https://support.getgrist.com/team-sharing/).\n - Control access to [individual rows, columns, and tables](https://support.getgrist.com/access-rules/).\n - Control access based on cell values and user attributes.\n * Self-maintainable.\n - Useful for intranet operation and specific compliance requirements.\n * Sandboxing options for untrusted documents.\n - On Linux or with Docker, you can enable [gVisor](https://github.com/google/gvisor) sandboxing at the individual document level.\n - On macOS, you can use native sandboxing.\n - On any OS, including Windows, you can use a wasm-based sandbox.\n * Translated to many languages.\n * `F1` key brings up some quick help. This used to go without saying, but in general Grist has good keyboard support.\n * We post progress on [𝕏 or Twitter or whatever](https://twitter.com/getgrist) and publish [monthly newsletters](https://support.getgrist.com/newsletters/).\n\nIf you are curious about where Grist is heading, see [our roadmap](https://github.com/gristlabs/grist-core/projects/1), drop a question in [our forum](https://community.getgrist.com), or browse [our extensive documentation](https://support.getgrist.com).\n\n## Features not in `grist-core`\n\nIf you evaluate Grist by using the hosted version at [getgrist.com](https://getgrist.com), be aware that it includes some extensions to Grist that aren't present in `grist-core`. To be sure you're seeing exactly what is present in `grist-core`, you can run the [desktop app](https://github.com/gristlabs/grist-desktop), or use [`docker`](#using-grist). Here is a list of features you may see in Grist Labs' hosting or Enterprise offerings that are not in `grist-core`, in chronological order of creation. If self-hosting, you can get access to a free trial of all of them using the Enterprise toggle on the [Admin Panel](https://support.getgrist.com/admin-panel/).\n\n * [GristConnect](https://support.getgrist.com/install/grist-connect/) (2022)\n - Any site that has plugins for letting Discourse use its logins (such as WordPress) can also let Grist use its logins.\n - GristConnect is a niche feature built for a specific client which you probably don't care about – `OIDC` and `SAML` support *is* part of `grist-core` and covers most authentication use cases.\n * [Azure back-end for document storage](https://support.getgrist.com/install/cloud-storage/#azure) (2022)\n - With `grist-core` you can store document versions in anything S3-compatible, which covers a lot of services, but not Azure specifically. The Azure back-end fills that gap.\n - Unless you are a Microsoft shop you probably don't care about this.\n * [Audit log streaming](https://support.getgrist.com/install/audit-log-streaming/) (2024)\n - With `grist-core` a lot of useful information is logged, but not organized specifically with auditing in mind. Audit log streaming supplies that organization, and a UI for setting things up.\n - Enterprises may care about this.\n * [Advanced Admin Controls](https://support.getgrist.com/admin-controls/) (2025)\n - This is a special page for a Grist installation administrator to monitor and edit user access to resources.\n - It uses a special set of administrative endpoints not present on `grist-core`.\n - If you're going to be running a large Grist installation, with employees coming and going, you may care about this.\n * [Grist Assistant](https://support.getgrist.com/assistant/#assistant) (2025)\n - An AI Formula Assistant - limited to working with formulas - is present in `grist-core`, but the newer Assistant can help with a wider range of tasks like building tables and dashboards and modifying data.\n - If you have many users who need help building documents or working with data, you may care about this one.\n * [Invite Notifications](https://support.getgrist.com/self-managed/#how-do-i-set-up-email-notifications) (2025)\n - When a user is added to a document, or a workspace, or a site, with email notifications they will get emailed a link to access the resource.\n - This link isn't special, with `grist-core` you can just send a link yourself or a colleague.\n - For a big Grist installation with users who aren't in close communication, emails might be nice? Hard to guess if you'll care about this one.\n * [Document Change and Comment Notifications](https://support.getgrist.com/document-settings/#notifications) (2025)\n - You can achieve change notifications in `grist-core` using webhooks, but it is less convenient.\n - People have been asking for this one for years. If you need an excuse to get your boss to pay for Grist, this might finally be the one that works?\n\n## Using Grist\n\nTo get the default version of `grist-core` running on your computer\nwith [Docker](https://www.docker.com/get-started), do:\n\n```sh\ndocker pull gristlabs/grist\ndocker run -p 8484:8484 -it gristlabs/grist\n```\n\nThen visit `http://localhost:8484` in your browser. You'll be able to create, edit, import,\nand export documents. To preserve your work across docker runs, share a directory as `/persist`:\n\n```sh\ndocker run -p 8484:8484 -v $PWD/persist:/persist -it gristlabs/grist\n```\n\nGet templates at [templates.getgrist.com](https://templates.getgrist.com) for payroll,\ninventory management, invoicing, D&D encounter tracking, and a lot\nmore, or use any document you've created on\n[docs.getgrist.com](https://docs.getgrist.com).\n\nIf you need to change the port Grist runs on, set a `PORT` variable, don't just change the\nport mapping:\n\n```\ndocker run --env PORT=9999 -p 9999:9999 -v $PWD/persist:/persist -it gristlabs/grist\n```\n\nTo enable gVisor sandboxing, set `--env GRIST_SANDBOX_FLAVOR=gvisor`.\nThis should work with default docker settings, but may not work in all\nenvironments.\n\nYou can find a lot more about configuring Grist, setting up authentication,\nand running it on a public server in our\n[Self-Managed Grist](https://support.getgrist.com/self-managed/) handbook.\n\n## Using Grist with OpenRouter for Model Agnostic and Claude Support\n\n(Instructions contributed by @lshalon)\n\nGrist's AI Formula Assistant can be configured to use OpenRouter instead of connecting directly to OpenAI, allowing you to access a wide range of AI models including Anthropic's Claude models. This isn't the only way to use Claude models, but it's a good option if you want to use Claude models with Grist or intend to use other cheaper, faster, or potentially newer models. That's because this configuration gives you more flexibility in choosing the AI model that works best for your formula generation needs.\nTo set up OpenRouter integration, configure the following environment variables:\n\n### Required: Set the endpoint to OpenRouter's API\n\n```\nASSISTANT_CHAT_COMPLETION_ENDPOINT=https://openrouter.ai/api/v1/chat/completions\n```\n\n### Required: Your OpenRouter API key\n\n```\nASSISTANT_API_KEY=your_openrouter_api_key_here\n```\n\nSign up for an OpenRouter API key at \n\n### Optional: Specify which model to use (examples below)\n\n```\nASSISTANT_MODEL=anthropic/claude-3.7-sonnet\n```\n\n### or other options like\n\n```\nASSISTANT_MODEL=deepseek/deepseek-r1-zero:free\n```\n\n```\nASSISTANT_MODEL=qwen/qwq-32b:free\n```\n\n```\nASSISTANT_MODEL=mistralai/mistral-saba\n```\n\n### Optional: Set a larger context model for fallback\n\n```\nASSISTANT_LONGER_CONTEXT_MODEL=anthropic/claude-3-opus-20240229\n```\n\nWith this configuration, Grist's AI Formula Assistant will route requests through OpenRouter to your specified model. This allows you to:\n\nAccess Anthropic's Claude models which excel at understanding context and generating accurate formulas\nSwitch between different AI models without changing your Grist configuration\nTake advantage of OpenRouter's routing capabilities to optimize for cost, speed, or quality\n\nYou can find the available models and their identifiers on the OpenRouter website.\nNote: Make sure not to set the OPENAI_API_KEY variable when using OpenRouter, as this would override the OpenRouter configuration.\n\n\n## Available Docker images\n\nThe default Docker image is `gristlabs/grist`. This contains all of\nthe standard Grist functionality, as well as extra source-available\ncode for enterprise customers taken from the\n[grist-ee](https://github.com/gristlabs/grist-ee) repository. This\nextra code is not under a free or open source license. By default,\nhowever, the code from the `grist-ee` repository is completely inert\nand inactive. This code becomes active only when enabled from the\nadministrator panel.\n\nIf you would rather use an image that contains exclusively free and\nopen source code, the `gristlabs/grist-oss` Docker image is available\nfor this purpose. It is by default functionally equivalent to the\n`gristlabs/grist` image.\n\n## The administrator panel\n\nYou can turn on a special admininistrator panel to inspect the status\nof your installation. Just visit `/admin` on your Grist server for\ninstructions. Since it is useful for the admin panel to be\navailable even when authentication isn't set up, you can give it a\nspecial access key by setting `GRIST_BOOT_KEY`.\n\n```\ndocker run -p 8484:8484 -e GRIST_BOOT_KEY=secret -it gristlabs/grist\n```\n\nThe boot page should then be available at\n`/admin?boot-key=`. We are collecting probes for\ncommon problems there. If you hit a problem that isn't covered, it\nwould be great if you could add a probe for it in\n[BootProbes](https://github.com/gristlabs/grist-core/blob/main/app/server/lib/BootProbes.ts).\nYou may instead file an issue so someone else can add it.\n\n## Building from source\n\nTo build Grist from source, follow these steps:\n\n yarn install\n yarn install:python\n yarn build\n yarn start\n # Grist will be available at http://localhost:8484/\n\nGrist formulas in documents will be run using Python executed directly on your\nmachine. You can configure sandboxing using a `GRIST_SANDBOX_FLAVOR`\nenvironment variable.\n\n * On macOS, `export GRIST_SANDBOX_FLAVOR=macSandboxExec`\n uses the native `sandbox-exec` command for sandboxing.\n * On Linux with [gVisor's runsc](https://github.com/google/gvisor)\n installed, `export GRIST_SANDBOX_FLAVOR=gvisor` is an option.\n * On any OS including Windows, `export GRIST_SANDBOX_FLAVOR=pyodide` is available.\n\nThese sandboxing methods have been written for our own use at Grist Labs and\nmay need tweaking to work in your own environment - pull requests\nvery welcome here!\n\nIf you wish to include Grist Labs enterprise extensions in your build,\nthe steps are as follows. Note that this will add non-OSS code to your\nbuild. It will also place a directory called `node_modules` one level\nup, at the same level as the Grist repo. If that is a problem for you,\njust move everything into a subdirectory first.\n\n yarn install\n yarn install:ee\n yarn install:python\n yarn build\n yarn start\n # Grist will be available at http://localhost:8484/\n\nThe enterprise code will by default not be used. You need to explicitly enable\nit in the [Admin Panel](https://support.getgrist.com/self-managed/#how-do-i-enable-grist-enterprise).\n\n## Logins\n\nLike git, Grist has features to track document revision history. So for full operation,\nGrist expects to know who the user modifying a document is. Until it does, it operates\nin a limited anonymous mode. To get you going, the docker image is configured so that\nwhen you click on the \"sign in\" button Grist will attribute your work to `you@example.com`.\nChange this by setting `GRIST_DEFAULT_EMAIL`:\n\n```\ndocker run --env GRIST_DEFAULT_EMAIL=my@email -p 8484:8484 -v $PWD/persist:/persist -it gristlabs/grist\n```\n\nYou can change your name in `Profile Settings` in\nthe [User Menu](https://support.getgrist.com/glossary/#user-menu).\n\nFor multi-user operation, or if you wish to access Grist across the\npublic internet, you'll want to connect it to your own Single Sign-On service.\nThere are a lot of ways to do this, including [SAML and forward authentication](https://support.getgrist.com/self-managed/#how-do-i-set-up-authentication).\nGrist has been tested with [Authentik](https://goauthentik.io/), [Auth0](https://auth0.com/),\nand Google/Microsoft sign-ins via [Dex](https://dexidp.io/).\n\n## Translations\n\nWe use [Weblate](https://hosted.weblate.org/engage/grist/) to manage translations.\nThanks to everyone who is pitching in. Thanks especially to the ANCT developers who\ndid the hard work of making a good chunk of the application localizable. Merci beaucoup !\n\n\n\"Translation\n\n\n[![Translation detail](https://hosted.weblate.org/widgets/grist/-/multi-green.svg)](https://hosted.weblate.org/engage/grist/)\n\n## Why free and open source software\n\nThis repository, `grist-core`, is maintained by Grist Labs. Our flagship product available at [getgrist.com](https://www.getgrist.com) is built from the code you see here, combined with business-specific software designed to scale to many users, handle billing, etc.\n\nGrist Labs is an open-core company. We offer Grist hosting as a service, with free and paid plans. We also develop and sell features related to Grist using a proprietary license, targeted at the needs of enterprises with large self-managed installations.\n\nWe see data portability and autonomy as a key value, and `grist-core` is an essential part of that. We are committed to maintaining and improving the `grist-core` codebase, and to be thoughtful about how proprietary offerings impact data portability and autonomy.\n\nBy opening its source code and offering an [OSI](https://opensource.org/)-approved free license, Grist benefits its users:\n\n- **Developer community.** The freedom to examine source code, make bug fixes, and develop\n new features is a big deal for a general-purpose spreadsheet-like product, where there is a\n very long tail of features vital to someone somewhere.\n- **Increased trust.** Because anyone can examine the source code, “security by obscurity” is not\n an option. Vulnerabilities in the code can be found by others and reported before they cause\n damage.\n- **Independence.** Grist is available to you regardless of the fortunes of the Grist Labs business,\n since it is open source and can be self-hosted. Using our hosted solution is convenient, but you\n are not locked in.\n- **Price flexibility.** If you are low on funds but have time to invest, self-hosting is a great\n option to have. And DIY users may have the technical savvy and motivation to delve in and make improvements,\n which can benefit all users of Grist.\n- **Extensibility.** For developers, having the source open makes it easier to build extensions (such as [Custom Widgets](https://support.getgrist.com/widget-custom/)). You can more easily include Grist in your pipeline. And if a feature is missing, you can just take the source code and build on top of it.\n\nFor more on Grist Labs' history and principles, see our [About Us](https://www.getgrist.com/about/) page.\n\n## Sponsors\n\n

\n \n \n \n

\n\n## Reviews\n\n * [Grist on ProductHunt](https://www.producthunt.com/posts/grist-2)\n * [Grist on AppSumo](https://appsumo.com/products/grist/) (life-time deal is sold out)\n * [Capterra](https://www.capterra.com/p/232821/Grist/#reviews), [G2](https://www.g2.com/products/grist/reviews), [TrustRadius](https://www.trustradius.com/products/grist/reviews)\n\n## Environment variables\n\nGrist can be configured in many ways. Here are the main environment variables it is sensitive to:\n\n| Variable | Purpose |\n| -------- | ------- |\n| ALLOWED_WEBHOOK_DOMAINS | comma-separated list of permitted domains to use in webhooks (e.g. webhook.site,zapier.com). You can set this to `*` to allow all domains, but if doing so, we recommend using a carefully locked-down proxy (see `GRIST_PROXY_FOR_UNTRUSTED_URLS`) if you do not entirely trust users. Otherwise services on your internal network may become vulnerable to manipulation. |\n| APP_DOC_URL | doc worker url, set when starting an individual doc worker (other servers will find doc worker urls via redis) |\n| APP_DOC_INTERNAL_URL | like `APP_DOC_URL` but used by the home server to reach the server using an internal domain name resolution (like in a docker environment). It only makes sense to define this value in the doc worker. Defaults to `APP_DOC_URL`. |\n| APP_HOME_URL | url prefix for home api (home and doc servers need this) |\n| APP_HOME_INTERNAL_URL | like `APP_HOME_URL` but used by the home and the doc servers to reach any home workers using an internal domain name resolution (like in a docker environment). Defaults to `APP_HOME_URL` |\n| APP_STATIC_URL | url prefix for static resources |\n| APP_STATIC_INCLUDE_CUSTOM_CSS | set to \"true\" to include custom.css (from APP_STATIC_URL) in static pages |\n| APP_UNTRUSTED_URL | URL at which to serve/expect plugin content. |\n| GRIST_ACTION_HISTORY_MAX_ROWS | Maximum number of rows allowed in ActionHistory before pruning (up to a 1.25 grace factor). Defaults to 1000. ⚠️ A too low value may make the \"[Work on a copy](https://support.getgrist.com/newsletters/2021-06/#work-on-a-copy)\" feature [malfunction](https://github.com/gristlabs/grist-core/issues/1121#issuecomment-2248112023) |\n| GRIST_ACTION_HISTORY_MAX_BYTES | Maximum number of rows allowed in ActionHistory before pruning (up to a 1.25 grace factor). Defaults to 1Gb. ⚠️ A too low value may make the \"[Work on a copy](https://support.getgrist.com/newsletters/2021-06/#work-on-a-copy)\" feature [malfunction](https://github.com/gristlabs/grist-core/issues/1121#issuecomment-2248112023) |\n| GRIST_ADAPT_DOMAIN | set to \"true\" to support multiple base domains (careful, host header should be trustworthy) |\n| GRIST_ALLOW_AUTOMATIC_VERSION_CHECKING | Whether Grist is allowed to automatically check if a newer Grist version is available. Defaults to \"true\" on the default `grist` and `grist-ee` Docker images. Defaults false in `grist-oss` and everywhere else. |\n| GRIST_ALLOW_DEPRECATED_BARE_ORG_DELETE | If set, the deprecated DELETE /api/orgs/:orgId endpoint is available. |\n| GRIST_APP_ROOT | directory containing Grist sandbox and assets (specifically the sandbox and static subdirectories). |\n| GRIST_ATTACHMENT_THRESHOLD_MB | attachment storage limit per document beyond which Grist will recommend external storage (if available). Defaults to 50MB. |\n| GRIST_BACKUP_DELAY_SECS | wait this long after a doc change before making a backup |\n| GRIST_BOOT_KEY | if set, offer diagnostics at /boot/GRIST_BOOT_KEY |\n| GRIST_BROADCAST_TIMEOUT_MS | Set the maximum time a web client has to accept a broadcast message about a document before being disconnected (default: 1 minute). |\n| GRIST_DATA_DIR | Directory in which to store documents. Defaults to `docs/` relative to the Grist application directory. In Grist's default Docker image, its default value is /persist/docs so that it will be used as a mounted volume. |\n| GRIST_DEFAULT_EMAIL | if set, login as this user if no other credentials presented |\n| GRIST_DEFAULT_PRODUCT | if set, this controls enabled features and limits of new sites. See names of PRODUCTS in Product.ts. |\n| GRIST_DEFAULT_LOCALE | Locale to use as fallback when Grist cannot honour the browser locale. |\n| GRIST_DOMAIN | in hosted Grist, Grist is served from subdomains of this domain. Defaults to \"getgrist.com\". |\n| GRIST_EXPERIMENTAL_PLUGINS | enables experimental plugins |\n| GRIST_EXTERNAL_ATTACHMENTS_MODE | required to enable external storage for attachments. Set to \"snapshots\" to enable external storage. Default value is \"none\". Note that when enabled, a [snapshot storage has to be configured](https://support.getgrist.com/self-managed/#how-do-i-set-up-snapshots) as well. |\n| GRIST_ENABLE_SERVICE_ACCOUNTS | enables the `service accounts` feature. This feature allows users to create special service accounts that they can manage and to whom they can grant restricted access to chosen resources. Useful as a way to get fine-grained api keys for use with third party automations. Unset by default |\n| GRIST_ENABLE_REQUEST_FUNCTION | enables the REQUEST function. This function performs HTTP requests in a similar way to `requests.request`. This function presents a significant security risk, since it can let users call internal endpoints when Grist is available publicly. This function can also cause performance issues. Unset by default. |\n| GRIST_HEADERS_TIMEOUT_MS | if set, override nodes's server.headersTimeout flag. |\n| GRIST_HIDE_UI_ELEMENTS | comma-separated list of UI features to disable. Allowed names of parts: `helpCenter`, `billing`, `templates`, `createSite`, `multiSite`, `multiAccounts`, `importFromAirtable`, `sendToDrive`, `tutorials`, `supportGrist`, `themes`. If a part also exists in GRIST_UI_FEATURES, it will still be disabled. |\n| GRIST_HOST | hostname to use when listening on a port. |\n| GRIST_PROXY_FOR_UNTRUSTED_URLS | Full URL of proxy for delivering webhook payloads. Default value is `direct` for delivering payloads without proxying. |\n| HTTPS_PROXY or https_proxy | Full URL of reverse web proxy (corporate proxy) for fetching the custom widgets repository or the OIDC config from the issuer. |\n| GRIST_ID_PREFIX | for subdomains of form o-*, expect or produce o-${GRIST_ID_PREFIX}*. |\n| GRIST_IGNORE_SESSION | if set, Grist will not use a session for authentication. |\n| GRIST_INCLUDE_CUSTOM_SCRIPT_URL | if set, will load the referenced URL in a `\n`;\n}\n" }, { "path": "app/common/TimeQuery.ts", "content": "import { concatenateSummaries } from \"app/common/ActionSummarizer\";\nimport { ActionSummary, ColumnDelta, createEmptyActionSummary, createEmptyTableDelta } from \"app/common/ActionSummary\";\nimport { CellDelta } from \"app/common/TabularDiff\";\n\nimport keyBy from \"lodash/keyBy\";\nimport matches from \"lodash/matches\";\nimport sortBy from \"lodash/sortBy\";\nimport toPairs from \"lodash/toPairs\";\n\n/**\n * We can combine an ActionSummary with the current state of the database\n * to answer questions about the state of the database in the past. This\n * is particularly useful for Grist metadata tables, which are needed to\n * interpret the content of user tables fully.\n * - TimeCursor is a simple container for the db and an ActionSummary\n * - TimeQuery offers a db-like interface for a given table and set of columns\n * - TimeLayout answers a couple of concrete questions about table meta-data using a\n * set of TimeQuery objects hooked up to _grist_* tables.\n */\n\nexport interface ResultRow {\n [column: string]: any;\n}\n\nexport interface ITimeData {\n fetch(tableId: string, colIds: string[], rowIds?: number[]): Promise;\n getColIds(tableId: string): Promise;\n}\n\n/** Track the state of the database at a particular time. */\nexport class TimeCursor {\n public summary: ActionSummary;\n\n constructor(public db: ITimeData) {\n this.summary = createEmptyActionSummary();\n }\n\n /**\n * Add a summary of an action just before the last action applied to\n * the TimeCursor, so we stretch further back in time.\n */\n public prepend(prevSummary: ActionSummary) {\n this.summary = concatenateSummaries([prevSummary, this.summary]);\n }\n\n /**\n * Add a summary of an action just after the last action applied to\n * the TimeCursor, going one step closer to current time. When the\n * cursor is used, the summary is assumed to extend right up to\n * current time.\n */\n public append(nextSummary: ActionSummary) {\n // TODO: concatenation appears to modify its inputs, so we\n // need to clone to avoid propagating that. Look to see if\n // a safe version of concatenation could be written to save\n // cloning.\n this.summary = concatenateSummaries([this.summary, nextSummary]);\n }\n}\n\n/** internal class for storing a ResultRow dictionary, keyed by rowId */\ninterface ResultRows {\n [rowId: number]: ResultRow;\n}\n\n/**\n * Query the state of a particular table in the past, given a TimeCursor holding the\n * current db and a summary of all changes between that past time and now.\n * For the moment, for simplicity, names of tables and columns are assumed not to\n * change, and TimeQuery should only be used for _grist_* tables.\n */\nexport class TimeQuery {\n private _currentRows: ResultRow[];\n private _pastRows: ResultRow[];\n\n constructor(public tc: TimeCursor,\n public tableId: string,\n public colIds: string[] | \"*\",\n public rowIds?: number[]) {\n }\n\n public reset(tableId: string, colIds: string[] | \"*\", rowIds?: number[]) {\n this.tableId = tableId;\n this.colIds = colIds;\n this.rowIds = rowIds;\n this._currentRows = [];\n this._pastRows = [];\n }\n\n /**\n * Get fresh data from DB and overlay with any past data.\n * TODO: optimize.\n */\n public async update(): Promise {\n this._currentRows = [];\n this._pastRows = [];\n\n const tableRenameDelta = this.tc.summary.tableRenames.find(\n delta => delta[0] === this.tableId,\n );\n const tableRenamed = tableRenameDelta ? tableRenameDelta[1] : this.tableId;\n // Table no longer exists.\n if (!tableRenamed) { return []; }\n\n // Let's see everything the summary has accumulated about the table back then.\n const td = this.tc.summary.tableDeltas[tableRenamed] || createEmptyTableDelta();\n\n const columnForwardRenames: Record =\n Object.fromEntries(td.columnRenames.filter(delta => delta[0]));\n const columnBackwardRenames: Record =\n Object.fromEntries(td.columnRenames.map(([a, b]) => [b, a]).filter(delta => delta[0]));\n\n const colIdsExpanded = this.colIds === \"*\" ?\n (await this.tc.db.getColIds(tableRenamed)).map(colId => columnBackwardRenames[colId] ?? colId) :\n this.colIds;\n\n const colIdsRenamed =\n colIdsExpanded.map(colId => columnForwardRenames[colId] ?? colId).filter(colId => colId);\n this._currentRows = await this.tc.db.fetch(\n tableRenamed,\n [\"id\", ...colIdsRenamed],\n this.rowIds,\n );\n\n // Now rewrite the summary as a ResultRow dictionary, to make it comparable\n // with database.\n const summaryRows: ResultRows = {};\n for (const [colId, columns] of toPairs(td.columnDeltas)) {\n for (const [rowId, cell] of toPairs(columns) as unknown as [keyof ColumnDelta, CellDelta][]) {\n if (!summaryRows[rowId]) { summaryRows[rowId] = {}; }\n const val = cell[0];\n summaryRows[rowId][colId] = (val !== null && typeof val === \"object\") ? val[0] : null;\n }\n }\n\n // Prepare to access the current database state by rowId.\n const rowsById = keyBy(this._currentRows, r => (r.id as number));\n\n // Prepare a list of rowIds at the time of interest.\n // The past rows are whatever the db has now, omitting rows that were added\n // since the past time, and adding back any rows that were removed since then.\n // Careful about the order of this, since rows could be replaced.\n const additions = new Set(td.addRows);\n const pastRowIds =\n new Set([...this._currentRows.map(r => r.id as number).filter(r => !additions.has(r)),\n ...td.removeRows]);\n\n // Now prepare a row for every expected rowId, using current db data if available\n // and relevant, and overlaying past data when available.\n this._pastRows = new Array();\n const colIdsOfInterest = new Set(colIdsExpanded);\n for (const id of Array.from(pastRowIds).sort()) {\n const rowCurrent: ResultRow = rowsById[id] || { id };\n const row: ResultRow = {};\n for (const colId of [\"id\", ...colIdsExpanded]) {\n const colIdRenamed = columnForwardRenames[colId] ?? colId;\n if (!colIdRenamed) { continue; }\n row[colId] = rowCurrent[colIdRenamed];\n }\n if (summaryRows[id] && !additions.has(id)) {\n for (const [colId, val] of toPairs(summaryRows[id])) {\n const colIdRenamed = columnBackwardRenames[colId] ?? colId;\n if (colIdsOfInterest.has(colIdRenamed)) {\n row[colIdRenamed] = val;\n }\n }\n }\n this._pastRows.push(row);\n }\n return this._pastRows;\n }\n\n /**\n * Do a query with a single result, specifying any desired filters. Exception thrown\n * if there is no result.\n */\n public one(args: { [name: string]: any }): ResultRow {\n const result = this._pastRows.find(matches(args));\n if (!result) {\n throw new Error(`could not find: ${JSON.stringify(args)} for ${this.tableId}`);\n }\n return result;\n }\n\n /** Get all results for a query. */\n public all(args?: { [name: string]: any }): ResultRow[] {\n if (!args) { return this._pastRows; }\n return this._pastRows.filter(matches(args));\n }\n}\n\n/**\n * Put some TimeQuery queries to work answering questions about column order and\n * user-facing name of tables.\n */\nexport class TimeLayout {\n public tables: TimeQuery;\n public fields: TimeQuery;\n public columns: TimeQuery;\n public views: TimeQuery;\n public sections: TimeQuery;\n\n constructor(public tc: TimeCursor) {\n this.tables = new TimeQuery(tc, \"_grist_Tables\", [\"tableId\", \"primaryViewId\", \"rawViewSectionRef\"]);\n this.fields = new TimeQuery(tc, \"_grist_Views_section_field\",\n [\"parentId\", \"parentPos\", \"colRef\"]);\n this.columns = new TimeQuery(tc, \"_grist_Tables_column\", [\"parentId\", \"colId\"]);\n this.views = new TimeQuery(tc, \"_grist_Views\", [\"id\", \"name\"]);\n this.sections = new TimeQuery(tc, \"_grist_Views_section\", [\"id\", \"title\"]);\n }\n\n /** update from TimeCursor */\n public async update() {\n await this.tables.update();\n await this.columns.update();\n await this.fields.update();\n await this.views.update();\n await this.sections.update();\n }\n\n public getColumnOrder(tableId: string): string[] {\n const primaryViewId = this.tables.one({ tableId }).primaryViewId;\n const preorder = this.fields.all({ parentId: primaryViewId });\n const precol = keyBy(this.columns.all(), \"id\");\n const ordered = sortBy(preorder, \"parentPos\");\n const names = ordered.map(r => precol[r.colRef].colId);\n return names;\n }\n\n public getTableName(tableId: string): string {\n const rawViewSectionRef = this.tables.one({ tableId }).rawViewSectionRef;\n return this.sections.one({ id: rawViewSectionRef }).title;\n }\n}\n" }, { "path": "app/common/Triggers-ti.ts", "content": "/**\n * This module was automatically generated by `ts-interface-builder`\n */\nimport * as t from \"ts-interface-checker\";\n// tslint:disable:object-literal-key-quotes\n\nexport const WebhookSubscribeCollection = t.iface([], {\n \"webhooks\": t.array(\"Webhook\"),\n});\n\nexport const Webhook = t.iface([], {\n \"fields\": \"WebhookFields\",\n});\n\nexport const WebhookFields = t.iface([], {\n \"url\": \"string\",\n \"authorization\": t.opt(\"string\"),\n \"eventTypes\": t.array(t.union(t.lit(\"add\"), t.lit(\"update\"))),\n \"tableId\": \"string\",\n \"watchedColIds\": t.opt(t.array(\"string\")),\n \"enabled\": t.opt(\"boolean\"),\n \"isReadyColumn\": t.opt(t.union(\"string\", \"null\")),\n \"condition\": t.opt(\"string\"),\n \"name\": t.opt(\"string\"),\n \"memo\": t.opt(\"string\"),\n});\n\nexport const WebhookBatchStatus = t.union(t.lit(\"success\"), t.lit(\"failure\"), t.lit(\"rejected\"));\n\nexport const WebhookStatus = t.union(t.lit(\"idle\"), t.lit(\"sending\"), t.lit(\"retrying\"), t.lit(\"postponed\"), t.lit(\"error\"), t.lit(\"invalid\"));\n\nexport const WebHookSecret = t.iface([], {\n \"url\": \"string\",\n \"unsubscribeKey\": \"string\",\n \"authorization\": t.opt(\"string\"),\n});\n\nexport const WebhookSubscribe = t.iface([], {\n \"url\": \"string\",\n \"authorization\": t.opt(\"string\"),\n \"eventTypes\": t.array(t.union(t.lit(\"add\"), t.lit(\"update\"))),\n \"watchedColIds\": t.opt(t.array(\"string\")),\n \"enabled\": t.opt(\"boolean\"),\n \"condition\": t.opt(\"string\"),\n \"isReadyColumn\": t.opt(t.union(\"string\", \"null\")),\n \"name\": t.opt(\"string\"),\n \"memo\": t.opt(\"string\"),\n});\n\nexport const WebhookSummaryCollection = t.iface([], {\n \"webhooks\": t.array(\"WebhookSummary\"),\n});\n\nexport const WebhookSummary = t.iface([], {\n \"id\": \"string\",\n \"fields\": t.iface([], {\n \"url\": \"string\",\n \"authorization\": t.opt(\"string\"),\n \"unsubscribeKey\": \"string\",\n \"eventTypes\": t.array(\"string\"),\n \"isReadyColumn\": t.union(\"string\", \"null\"),\n \"tableId\": \"string\",\n \"watchedColIds\": t.opt(t.array(\"string\")),\n \"enabled\": \"boolean\",\n \"name\": \"string\",\n \"memo\": \"string\",\n }),\n \"usage\": t.union(\"WebhookUsage\", \"null\"),\n});\n\nexport const WebhookUpdate = t.iface([], {\n \"id\": \"string\",\n \"fields\": \"WebhookPatch\",\n});\n\nexport const WebhookPatch = t.iface([], {\n \"url\": t.opt(\"string\"),\n \"authorization\": t.opt(\"string\"),\n \"eventTypes\": t.opt(t.array(t.union(t.lit(\"add\"), t.lit(\"update\")))),\n \"tableId\": t.opt(\"string\"),\n \"watchedColIds\": t.opt(t.array(\"string\")),\n \"enabled\": t.opt(\"boolean\"),\n \"isReadyColumn\": t.opt(t.union(\"string\", \"null\")),\n \"name\": t.opt(\"string\"),\n \"memo\": t.opt(\"string\"),\n});\n\nexport const WebhookUsage = t.iface([], {\n \"numWaiting\": \"number\",\n \"status\": \"WebhookStatus\",\n \"updatedTime\": t.opt(t.union(\"number\", \"null\")),\n \"lastSuccessTime\": t.opt(t.union(\"number\", \"null\")),\n \"lastFailureTime\": t.opt(t.union(\"number\", \"null\")),\n \"lastErrorMessage\": t.opt(t.union(\"string\", \"null\")),\n \"lastHttpStatus\": t.opt(t.union(\"number\", \"null\")),\n \"lastEventBatch\": t.opt(t.union(\"null\", t.iface([], {\n \"size\": \"number\",\n \"errorMessage\": t.union(\"string\", \"null\"),\n \"httpStatus\": t.union(\"number\", \"null\"),\n \"status\": \"WebhookBatchStatus\",\n \"attempts\": \"number\",\n }))),\n \"numSuccess\": t.opt(t.iface([], {\n \"pastHour\": \"number\",\n \"past24Hours\": \"number\",\n })),\n});\n\nexport const TriggerAction = t.union(\"WebhookAction\", \"EmailAction\");\n\nexport const WebhookAction = t.iface([], {\n \"type\": t.lit(\"webhook\"),\n \"id\": \"string\",\n});\n\nexport const EmailAction = t.iface([], {\n \"id\": \"string\",\n \"type\": t.lit(\"email\"),\n \"to\": \"string\",\n \"subject\": \"string\",\n \"body\": \"string\",\n});\n\nconst exportedTypeSuite: t.ITypeSuite = {\n WebhookSubscribeCollection,\n Webhook,\n WebhookFields,\n WebhookBatchStatus,\n WebhookStatus,\n WebHookSecret,\n WebhookSubscribe,\n WebhookSummaryCollection,\n WebhookSummary,\n WebhookUpdate,\n WebhookPatch,\n WebhookUsage,\n TriggerAction,\n WebhookAction,\n EmailAction,\n};\nexport default exportedTypeSuite;\n" }, { "path": "app/common/Triggers.ts", "content": "export interface WebhookSubscribeCollection {\n webhooks: Webhook[]\n}\n\nexport interface Webhook {\n fields: WebhookFields;\n}\n\nexport interface WebhookFields {\n url: string;\n authorization?: string;\n eventTypes: (\"add\" | \"update\")[];\n tableId: string;\n watchedColIds?: string[];\n enabled?: boolean;\n isReadyColumn?: string | null;\n condition?: string;\n name?: string;\n memo?: string;\n}\n\n// Union discriminated by type\nexport type WebhookBatchStatus = \"success\" | \"failure\" | \"rejected\";\nexport type WebhookStatus = \"idle\" | \"sending\" | \"retrying\" | \"postponed\" | \"error\" | \"invalid\";\n\n/** Secrets for webhook stored outside the document in home db */\nexport interface WebHookSecret {\n url: string;\n unsubscribeKey: string;\n authorization?: string;\n}\n\n// WebhookSubscribe should be `Omit` (because subscribe endpoint read\n// tableId from the url) but generics are not yet supported by ts-interface-builder\nexport interface WebhookSubscribe {\n url: string;\n authorization?: string;\n eventTypes: (\"add\" | \"update\")[];\n watchedColIds?: string[];\n enabled?: boolean;\n condition?: string;\n isReadyColumn?: string | null;\n name?: string;\n memo?: string;\n}\n\nexport interface WebhookSummaryCollection {\n webhooks: WebhookSummary[];\n}\nexport interface WebhookSummary {\n id: string;\n fields: {\n url: string;\n authorization?: string;\n unsubscribeKey: string;\n eventTypes: string[];\n isReadyColumn: string | null;\n tableId: string;\n watchedColIds?: string[];\n enabled: boolean;\n name: string;\n memo: string;\n },\n usage: WebhookUsage | null,\n}\n\n// Describes fields to update a webhook\nexport interface WebhookUpdate {\n id: string;\n fields: WebhookPatch;\n}\n\n// WebhookPatch should be `Partial` but generics are not yet supported by\n// ts-interface-builder\nexport interface WebhookPatch {\n url?: string;\n authorization?: string;\n eventTypes?: (\"add\" | \"update\")[];\n tableId?: string;\n watchedColIds?: string[];\n enabled?: boolean;\n isReadyColumn?: string | null;\n name?: string;\n memo?: string;\n}\n\nexport interface WebhookUsage {\n // As minimum we need number of waiting events and status (by default pending).\n numWaiting: number,\n status: WebhookStatus;\n updatedTime?: number | null;\n lastSuccessTime?: number | null;\n lastFailureTime?: number | null;\n lastErrorMessage?: string | null;\n lastHttpStatus?: number | null;\n lastEventBatch?: null | {\n size: number;\n errorMessage: string | null;\n httpStatus: number | null;\n status: WebhookBatchStatus;\n attempts: number;\n },\n numSuccess?: {\n pastHour: number;\n past24Hours: number;\n },\n}\n\n// Union type for trigger actions. Currently only WebhookAction is supported, but this is\n// designed as a discriminated union to support additional action types in the future (e.g., emails).\nexport type TriggerAction = WebhookAction | EmailAction;\n\nexport interface WebhookAction {\n // The type field is used to discriminate between different action types.\n // For now we have only webhook, but next types in the pipeline are emails.\n type: \"webhook\";\n id: string; // Unique id of the action, used as a key in homeDB secrets for webhooks\n}\n\nexport interface EmailAction {\n id: string;\n type: \"email\";\n to: string; // Comma-separated list of email addresses, user refs.\n subject: string;\n body: string;\n}\n" }, { "path": "app/common/User.ts", "content": "import { getTableId } from \"app/common/DocActions\";\nimport { EmptyRecordView, RecordView } from \"app/common/RecordView\";\nimport { Role } from \"app/common/roles\";\n\n/**\n * User type to distinguish between Users and service accounts\n */\nexport type UserType = \"login\" | \"service\";\n\n/**\n * Information about a user, including any user attributes.\n */\nexport interface UserInfo {\n Name: string | null;\n Email: string | null;\n Access: Role | null;\n Origin: string | null;\n LinkKey: Record;\n UserID: number | null;\n UserRef: string | null;\n SessionID: string | null;\n /**\n * This is a rowId in the _grist_Shares table, if the user is accessing a document\n * via a share. Otherwise null.\n */\n ShareRef: number | null;\n Type: UserType | null;\n [attributes: string]: unknown;\n}\n\n/**\n * Wrapper class for `UserInfo`.\n *\n * Contains methods for converting itself to different representations.\n */\nexport class User implements UserInfo {\n public Name: string | null = null;\n public UserID: number | null = null;\n public Access: Role | null = null;\n public Origin: string | null = null;\n public LinkKey: Record = {};\n public Email: string | null = null;\n public SessionID: string | null = null;\n public UserRef: string | null = null;\n public ShareRef: number | null = null;\n public Type: UserType | null = null;\n [attribute: string]: any;\n\n constructor(info: Record = {}) {\n Object.assign(this, info);\n }\n\n /**\n * Returns a JSON representation of this class that excludes full row data,\n * only keeping user info and table/row ids for any user attributes.\n *\n * Used by the sandbox to support `user` variables in formulas (see `user.py`).\n */\n public toJSON() {\n return this._toObject((value) => {\n if (value instanceof RecordView) {\n return [getTableId(value.data), value.get(\"id\")];\n } else if (value instanceof EmptyRecordView) {\n return null;\n } else {\n return value;\n }\n });\n }\n\n /**\n * Returns a record representation of this class, with all user attributes\n * converted from `RecordView` instances to their JSON representations.\n *\n * Used by the client to support `user` variables in dropdown conditions.\n */\n public toUserInfo(): UserInfo {\n return this._toObject((value) => {\n if (value instanceof RecordView) {\n return value.toJSON();\n } else if (value instanceof EmptyRecordView) {\n return null;\n } else {\n return value;\n }\n }) as UserInfo;\n }\n\n private _toObject(mapValue: (value: unknown) => unknown) {\n const results: { [key: string]: any } = {};\n for (const [key, value] of Object.entries(this)) {\n results[key] = mapValue(value);\n }\n return results;\n }\n}\n" }, { "path": "app/common/UserAPI.ts", "content": "import { ApplyUAResult, ForkResult, FormulaTimingInfo,\n PermissionDataWithExtraUsers, QueryFilters, TimingStatus } from \"app/common/ActiveDocAPI\";\nimport { AssistanceRequest, AssistanceResponse } from \"app/common/Assistance\";\nimport { BaseAPI, IOptions } from \"app/common/BaseAPI\";\nimport { BillingAPI, BillingAPIImpl } from \"app/common/BillingAPI\";\nimport { BrowserSettings } from \"app/common/BrowserSettings\";\nimport { ICustomWidget } from \"app/common/CustomWidget\";\nimport { BulkColValues, TableColValues, TableRecordValue, TableRecordValues,\n TableRecordValuesWithoutIds, UserAction } from \"app/common/DocActions\";\nimport { DocCreationInfo, OpenDocMode } from \"app/common/DocListAPI\";\nimport { DocStateComparison, DocStates } from \"app/common/DocState\";\nimport { OrgUsageSummary } from \"app/common/DocUsage\";\nimport { Features, Product } from \"app/common/Features\";\nimport { isClient } from \"app/common/gristUrls\";\nimport { encodeQueryParams } from \"app/common/gutil\";\nimport { FullUser, UserProfile } from \"app/common/LoginSessionAPI\";\nimport { OrgPrefs, UserOrgPrefs, UserPrefs } from \"app/common/Prefs\";\nimport * as roles from \"app/common/roles\";\nimport { StringUnion } from \"app/common/StringUnion\";\nimport {\n WebhookFields,\n WebhookSubscribe,\n WebhookSummaryCollection,\n WebhookUpdate,\n} from \"app/common/Triggers\";\nimport { addCurrentOrgToPath, getGristConfig } from \"app/common/urlUtils\";\nimport { AttachmentStore, AttachmentStoreDesc, TablesGet } from \"app/plugin/DocApiTypes\";\n\nimport { AxiosProgressEvent } from \"axios\";\nimport omitBy from \"lodash/omitBy\";\n\nexport type { FullUser, UserProfile };\n\n// Nominal email address of the anonymous user.\nexport const ANONYMOUS_USER_EMAIL = \"anon@getgrist.com\";\n\n// Nominal email address of a user who, if you share with them, everyone gets access.\nexport const EVERYONE_EMAIL = \"everyone@getgrist.com\";\n\n// Nominal email address of a user who can view anything (for thumbnails).\nexport const PREVIEWER_EMAIL = \"thumbnail@getgrist.com\";\n\n// A special 'docId' that means to create a new document.\nexport const NEW_DOCUMENT_CODE = \"new\";\n\n// Properties shared by org, workspace, and doc resources.\nexport interface CommonProperties {\n name: string;\n createdAt: string; // ISO date string\n updatedAt: string; // ISO date string\n removedAt?: string; // ISO date string - only can appear on docs and workspaces currently\n disabledAt?: string; // ISO date string - only can appear on docs currently\n public?: boolean; // If set, resource is available to the public\n}\nexport const commonPropertyKeys = [\"createdAt\", \"name\", \"updatedAt\"];\n\nexport interface OrganizationProperties extends CommonProperties {\n domain: string | null;\n // Organization includes preferences relevant to interacting with its content.\n userOrgPrefs?: UserOrgPrefs; // Preferences specific to user and org\n orgPrefs?: OrgPrefs; // Preferences specific to org (but not a particular user)\n userPrefs?: UserPrefs; // Preferences specific to user (but not a particular org)\n}\nexport const organizationPropertyKeys = [...commonPropertyKeys, \"domain\",\n \"orgPrefs\", \"userOrgPrefs\", \"userPrefs\"];\n\n// Basic information about an organization, excluding the user's access level\nexport interface OrganizationWithoutAccessInfo extends OrganizationProperties {\n id: number;\n owner: FullUser | null;\n billingAccount?: BillingAccount;\n host: string | null; // if set, org's preferred domain (e.g. www.thing.com)\n}\n\n// Organization information plus the user's access level\nexport interface Organization extends OrganizationWithoutAccessInfo {\n access: roles.Role;\n}\n\n// Basic information about a billing account associated with an org or orgs.\nexport interface BillingAccount {\n id: number;\n individual: boolean;\n product: Product;\n stripePlanId: string; // Stripe price id.\n isManager: boolean;\n inGoodStanding: boolean;\n features?: Features; // Features override, not the final set of features.\n externalOptions?: {\n invoiceId?: string;\n };\n}\n\n// The upload types vary based on which fetch implementation is in use. This is\n// an incomplete list. For example, node streaming types are supported by node-fetch.\nexport type UploadType = string | Blob | Buffer;\n\n/**\n * Returns a user-friendly org name, which is either org.name, or \"@User Name\" for personal orgs.\n */\nexport function getOrgName(org: Organization): string {\n return org.owner ? `@` + org.owner.name : org.name;\n}\n\n/**\n * Returns whether the given org is the templates org, which contains the public\n * templates and tutorials.\n */\nexport function isTemplatesOrg(org: { domain: Organization[\"domain\"] } | null): boolean {\n if (!org) { return false; }\n\n const { templateOrg } = getGristConfig();\n return org.domain === templateOrg;\n}\n\nexport type WorkspaceProperties = CommonProperties;\nexport const workspacePropertyKeys = [\"createdAt\", \"name\", \"updatedAt\"];\n\nexport interface Workspace extends WorkspaceProperties {\n id: number;\n docs: Document[];\n org: Organization;\n orgDomain?: string;\n access: roles.Role;\n owner?: FullUser; // Set when workspaces are in the \"docs\" pseudo-organization,\n // assembled from multiple personal organizations.\n // Not set when workspaces are all from the same organization.\n\n // Set when the workspace belongs to support@getgrist.com. We expect only one such workspace\n // (\"Examples & Templates\"), containing sample documents.\n isSupportWorkspace?: boolean;\n}\n\n// null stands for normal document type, the one set by default at document creation.\nexport const DOCTYPE_NORMAL = null;\nexport const DOCTYPE_TEMPLATE = \"template\";\nexport const DOCTYPE_TUTORIAL = \"tutorial\";\n\nexport type DocumentType = typeof DOCTYPE_NORMAL | typeof DOCTYPE_TEMPLATE | typeof DOCTYPE_TUTORIAL;\n\n// Non-core options for a document.\n// \"Non-core\" means bundled into a single options column in the database.\n// TODO: consider smoothing over this distinction in the API.\nexport interface DocumentOptions {\n description?: string | null;\n icon?: string | null;\n openMode?: OpenDocMode | null;\n externalId?: string | null; // A slot for storing an externally maintained id.\n // Not used in grist-core, but handy for Electron app.\n tutorial?: TutorialMetadata | null;\n appearance?: DocumentAppearance | null;\n // Whether search engines should index this document. Defaults to `false`.\n allowIndex?: boolean;\n proposedChanges?: ProposedChanges | null;\n}\n\nexport interface TutorialMetadata {\n lastSlideIndex?: number;\n percentComplete?: number;\n}\n\ninterface DocumentAppearance {\n icon?: DocumentIcon | null;\n}\n\ninterface DocumentIcon {\n backgroundColor?: string;\n color?: string;\n emoji?: string | null;\n}\n\nexport interface DocumentProperties extends CommonProperties {\n isPinned: boolean;\n urlId: string | null;\n trunkId: string | null;\n type: DocumentType | null;\n options: DocumentOptions | null;\n}\n\nexport interface ProposedChanges {\n mayHaveProposals?: boolean;\n acceptProposals?: boolean;\n}\n\nexport const documentPropertyKeys = [\n ...commonPropertyKeys,\n \"isPinned\",\n \"urlId\",\n \"options\",\n \"type\",\n \"appearance\",\n];\n\nexport interface Document extends DocumentProperties {\n id: string;\n workspace: Workspace;\n access: roles.Role;\n trunkAccess?: roles.Role | null;\n forks?: Fork[];\n}\n\nexport interface Fork {\n id: string;\n trunkId: string;\n updatedAt: string; // ISO date string\n options: DocumentOptions | null;\n}\n\nexport interface ProposalComparison {\n comparison?: DocStateComparison;\n}\n\nexport interface ProposalStatus {\n status?: \"applied\" | \"retracted\" | \"dismissed\";\n}\n\nexport interface Proposal {\n shortId: number;\n comparison: ProposalComparison;\n status: ProposalStatus;\n createdAt: string; // ISO date string\n updatedAt: string; // ISO date string\n appliedAt: string | null; // ISO date string\n srcDocId: string;\n srcDoc: Document & {\n creator: FullUser\n },\n destDocId: string;\n destDoc: Document & {\n creator: FullUser\n },\n}\n\n// Non-core options for a user.\nexport interface UserOptions {\n // Whether signing in with Google is allowed. Defaults to true if unset.\n allowGoogleLogin?: boolean;\n // The \"sub\" (subject) from the JWT issued by the password-based authentication provider.\n authSubject?: string;\n // Whether user is a consultant. Consultant users can be added to sites\n // without being counted for billing. Defaults to false if unset.\n isConsultant?: boolean;\n // Locale selected by the user. Defaults to 'en' if unset.\n locale?: string;\n ssoExtraInfo?: Record; // Extra fields from the user profile, e.g. from OIDC.\n // The first time the user logged in using getgrist.com auth. Only set in Grist SaaS.\n firstOAuthLoginAt?: Date;\n}\n\nexport interface PermissionDelta {\n maxInheritedRole?: roles.BasicRole | null;\n users?: {\n // Maps from email to group name, or null to inherit.\n [email: string]: roles.NonGuestRole | null\n };\n}\n\nexport interface PermissionData {\n // True if permission data is restricted to current user.\n personal?: true;\n // True if current user is a public member.\n public?: boolean;\n maxInheritedRole?: roles.BasicRole | null;\n users: UserAccessData[];\n}\n\n// A structure for modifying managers of a billing account.\nexport interface ManagerDelta {\n users: {\n // To add a manager, link their email to 'managers'.\n // To remove a manager, link their email to null.\n // This format is used to rhyme with the ACL PermissionDelta format.\n [email: string]: \"managers\" | null\n };\n}\n\nexport interface UserAccess {\n // Represents the user's direct access to the resource of interest. Lack of access to a resource\n // is represented by a null value.\n access: roles.Role | null;\n // A user's parentAccess represent their effective inheritable access to the direct parent of the resource\n // of interest. The user's effective access to the resource of interest can be determined based\n // on the user's parentAccess, the maxInheritedRole setting of the resource and the user's direct\n // access to the resource. Lack of access to the parent resource is represented by a null value.\n // If parent has non-inheritable access, this should be null.\n parentAccess?: roles.BasicRole | null;\n}\n\n// Information about a user and their access to an unspecified resource of interest.\nexport interface UserAccessData extends UserAccess {\n id: number;\n name: string;\n email: string;\n ref?: string | null;\n picture?: string | null; // When present, a url to a public image of unspecified dimensions.\n orgAccess?: roles.BasicRole | null;\n anonymous?: boolean; // If set to true, the user is the anonymous user.\n isMember?: boolean;\n disabledAt?: Date | null; // If not null, the user is disabled\n}\n\n/**\n * Combines access, parentAccess, and maxInheritedRole info into the resulting access role.\n */\nexport function getRealAccess(\n user: UserAccess, inherited: { maxInheritedRole?: roles.BasicRole | null },\n): roles.Role | null {\n const inheritedAccess = roles.getWeakestRole(user.parentAccess || null, inherited.maxInheritedRole || null);\n return roles.getStrongestRole(user.access, inheritedAccess);\n}\n\nconst roleNames: { [role: string]: string } = {\n [roles.OWNER]: \"Owner\",\n [roles.EDITOR]: \"Editor\",\n [roles.VIEWER]: \"Viewer\",\n};\n\nexport function getUserRoleText(user: UserAccessData) {\n return roleNames[user.access!] || user.access || \"no access\";\n}\n\nexport interface ExtendedUser extends FullUser {\n helpScoutSignature?: string;\n isInstallAdmin?: boolean; // Set if user is allowed to manage this installation.\n}\n\nexport interface ActiveSessionInfo {\n user: ExtendedUser;\n org: Organization | null;\n orgError?: OrgError;\n}\n\nexport interface OrgError {\n error: string;\n status: number;\n}\n\n/**\n * Options to control the source of a document being replaced. For\n * example, a document could be initialized from another document\n * (e.g. a fork) or from a snapshot.\n */\nexport interface DocReplacementOptions {\n /**\n * The docId to copy from.\n */\n sourceDocId?: string;\n /**\n * The s3 version ID.\n */\n snapshotId?: string;\n /**\n * True if tutorial metadata should be reset.\n *\n * Metadata that's reset includes the doc (i.e. tutorial) name, and the\n * properties under options.tutorial (e.g. lastSlideIndex).\n */\n resetTutorialMetadata?: boolean;\n}\n\n/**\n * Information about a single document snapshot/backup.\n */\nexport interface DocSnapshot {\n lastModified: string; // when the snapshot was made\n snapshotId: string; // the id of the snapshot in the underlying store\n docId: string; // an id for accessing the snapshot as a Grist document\n}\n\n/**\n * A list of document snapshots.\n */\nexport interface DocSnapshots {\n snapshots: DocSnapshot[]; // snapshots, freshest first.\n}\n\nexport interface CopyDocOptions {\n documentName: string;\n asTemplate?: boolean;\n}\n\nexport interface RenameDocOptions {\n icon?: DocumentIcon | null;\n}\n\nexport interface UserAPI {\n getSessionActive(): Promise;\n setSessionActive(email: string, org?: string): Promise;\n getSessionAll(): Promise<{ users: FullUser[], orgs: Organization[] }>;\n getOrgs(merged?: boolean): Promise;\n getWorkspace(workspaceId: number): Promise;\n getOrg(orgId: number | string): Promise;\n getOrgWorkspaces(orgId: number | string, includeSupport?: boolean): Promise;\n getOrgUsageSummary(orgId: number | string): Promise;\n getTemplates(): Promise;\n getTemplate(docId: string): Promise;\n getDoc(docId: string): Promise;\n newOrg(props: Partial): Promise;\n newWorkspace(props: Partial, orgId: number | string): Promise;\n newDoc(props: Partial, workspaceId: number): Promise;\n newUnsavedDoc(options?: { timezone?: string }): Promise;\n copyDoc(sourceDocumentId: string, workspaceId: number, options: CopyDocOptions): Promise;\n renameOrg(orgId: number | string, name: string): Promise;\n renameWorkspace(workspaceId: number, name: string): Promise;\n renameDoc(docId: string, name: string, options?: RenameDocOptions): Promise;\n updateOrg(orgId: number | string, props: Partial): Promise;\n updateDoc(docId: string, props: Partial): Promise;\n deleteOrg(orgId: number | string): Promise;\n deleteWorkspace(workspaceId: number): Promise; // delete workspace permanently\n softDeleteWorkspace(workspaceId: number): Promise; // soft-delete workspace\n undeleteWorkspace(workspaceId: number): Promise; // recover soft-deleted workspace\n deleteDoc(docId: string): Promise; // delete doc permanently\n softDeleteDoc(docId: string): Promise; // soft-delete doc\n undeleteDoc(docId: string): Promise; // recover soft-deleted doc\n disableDoc(docId: string): Promise; // (admin-only) remove all access to doc except deletion\n enableDoc(docId: string): Promise; // (admin-only) recover disabled doc\n updateOrgPermissions(orgId: number | string, delta: PermissionDelta): Promise;\n updateWorkspacePermissions(workspaceId: number, delta: PermissionDelta): Promise;\n updateDocPermissions(docId: string, delta: PermissionDelta): Promise;\n getOrgAccess(orgId: number | string): Promise;\n getWorkspaceAccess(workspaceId: number): Promise;\n getDocAccess(docId: string): Promise;\n pinDoc(docId: string): Promise;\n unpinDoc(docId: string): Promise;\n moveDoc(docId: string, workspaceId: number): Promise;\n getUserProfile(): Promise;\n updateUserName(name: string): Promise;\n updateUserLocale(locale: string | null): Promise;\n updateAllowGoogleLogin(allowGoogleLogin: boolean): Promise;\n disableUser(userId: number): Promise;\n enableUser(userId: number): Promise;\n updateIsConsultant(userId: number, isConsultant: boolean): Promise;\n getWorker(key: string): Promise;\n getWorkerFull(key: string): Promise;\n getWorkerAPI(key: string): Promise;\n getBillingAPI(): BillingAPI;\n getDocAPI(docId: string): DocAPI;\n fetchApiKey(): Promise;\n createApiKey(): Promise;\n deleteApiKey(): Promise;\n getTable(docId: string, tableName: string): Promise;\n applyUserActions(docId: string, actions: UserAction[]): Promise;\n importUnsavedDoc(material: UploadType, options?: {\n filename?: string,\n timezone?: string,\n onUploadProgress?: (ev: AxiosProgressEvent) => void,\n }): Promise;\n deleteUser(userId: number, name: string): Promise;\n getBaseUrl(): string; // Get the prefix for all the endpoints this object wraps.\n forRemoved(): UserAPI; // Get a version of the API that works on removed resources.\n getWidgets(): Promise;\n /**\n * Deletes account and personal org with all documents. Note: deleteUser doesn't clear documents, and this method\n * is specific to Grist installation, and might not be supported. Pass current user's id so that we can verify\n * that the user is deleting their own account. This is just to prevent accidental deletion from multiple tabs.\n *\n * @returns true if the account was deleted, false if there was a mismatch with the current user's id, and the\n * account was probably already deleted.\n */\n closeAccount(userId: number): Promise;\n /**\n * Deletes current non personal org with all documents. Note: deleteOrg doesn't clear documents, and this method\n * is specific to Grist installation, and might not be supported.\n */\n closeOrg(): Promise;\n}\n\n/**\n * Parameters for the download CSV and XLSX endpoint (/download/table-schema & /download/csv & /download/csv).\n */\nexport interface DownloadDocParams {\n tableId: string;\n viewSection?: number;\n activeSortSpec?: string;\n filters?: string;\n}\n\nexport const CreatableArchiveFormats = StringUnion(\"zip\", \"tar\");\nexport type CreatableArchiveFormats = typeof CreatableArchiveFormats.type;\n\nexport interface AttachmentsArchiveParams {\n format?: CreatableArchiveFormats,\n}\n\nexport interface ArchiveUploadResult {\n added: number;\n errored: number;\n unused: number;\n}\n\ninterface GetRowsParams {\n filters?: QueryFilters;\n immediate?: boolean;\n}\n\ninterface SqlResult extends TableRecordValuesWithoutIds {\n statement: string;\n}\n\nexport const DocAttachmentsLocation = StringUnion(\n \"none\", \"internal\", \"mixed\", \"external\",\n);\nexport type DocAttachmentsLocation = typeof DocAttachmentsLocation.type;\n\nexport const ExpandTableOption = StringUnion(\"column\");\nexport type ExpandTableOption = typeof ExpandTableOption.type;\n\ninterface GetTablesParams {\n expand?: ExpandTableOption[];\n}\n\n/**\n * Collect endpoints related to the content of a single document that we've been thinking\n * of as the (restful) \"Doc API\". A few endpoints that could be here are not, for historical\n * reasons, such as downloads.\n */\nexport interface DocAPI {\n readonly options: IOptions;\n getBaseUrl(): string;\n getTables(options?: GetTablesParams): Promise;\n // Immediate flag is a currently not-advertised feature, allowing a query to proceed without\n // waiting for a document to be initialized. This is useful if the calculations done when\n // opening a document are irrelevant.\n getRows(tableId: string, options?: GetRowsParams): Promise;\n getRecords(tableId: string, options?: GetRowsParams): Promise;\n sql(sql: string, args?: any[]): Promise;\n updateRows(tableId: string, changes: TableColValues): Promise;\n addRows(tableId: string, additions: BulkColValues): Promise;\n removeRows(tableId: string, removals: number[]): Promise;\n fork(): Promise;\n replace(source: DocReplacementOptions): Promise;\n // Get list of document versions (specify raw to bypass caching, which should only make\n // a difference if snapshots have \"leaked\")\n getSnapshots(raw?: boolean): Promise;\n // remove selected snapshots, or all snapshots that have \"leaked\" from inventory (should\n // be empty), or all but the current snapshot.\n removeSnapshots(snapshotIds: string[] | \"unlisted\" | \"past\"): Promise<{ snapshotIds: string[] }>;\n getStates(): Promise;\n forceReload(): Promise;\n recover(recoveryMode: boolean): Promise;\n // Compare two documents, optionally including details of the changes.\n compareDoc(\n remoteDocId: string,\n options?: { detail?: boolean; maxRows?: number | null }\n ): Promise;\n // Compare two versions within a document, including details of the changes.\n // Versions are identified by action hashes, or aliases understood by HashUtil.\n // Currently, leftHash is expected to be an ancestor of rightHash. If rightHash\n // is HEAD, the result will contain a copy of any rows added or updated.\n compareVersion(leftHash: string, rightHash: string): Promise;\n getDownloadUrl(options: { template: boolean, removeHistory: boolean }): string;\n getDownloadXlsxUrl(params?: DownloadDocParams): string;\n getDownloadCsvUrl(params: DownloadDocParams): string;\n getDownloadTsvUrl(params: DownloadDocParams): string;\n getDownloadDsvUrl(params: DownloadDocParams): string;\n getDownloadTableSchemaUrl(params: DownloadDocParams): string;\n getDownloadAttachmentsArchiveUrl(params: AttachmentsArchiveParams): string;\n\n /**\n * Exports current document to the Google Drive as a spreadsheet file. To invoke this method, first\n * acquire \"code\" via Google Auth Endpoint (see ShareMenu.ts for an example).\n * @param code Authorization code returned from Google (requested via Grist's Google Auth Endpoint)\n * @param title Name of the spreadsheet that will be created (should use a Grist document's title)\n */\n sendToDrive(code: string, title: string): Promise<{ url: string }>;\n // Upload a single attachment and return the resulting metadata row ID.\n // The arguments are passed to FormData.append.\n uploadAttachment(value: string | Blob, filename?: string): Promise;\n uploadAttachmentArchive(archive: string | Blob, filename?: string): Promise;\n\n // Get users that are worth proposing to \"View As\" for access control purposes.\n getUsersForViewAs(): Promise;\n\n getWebhooks(): Promise;\n addWebhook(webhook: WebhookFields): Promise<{ webhookId: string }>;\n removeWebhook(webhookId: string, tableId: string): Promise;\n // Update webhook\n updateWebhook(webhook: WebhookUpdate): Promise;\n flushWebhooks(): Promise;\n flushWebhook(webhookId: string): Promise;\n\n getAssistance(params: AssistanceRequest): Promise;\n /**\n * Check if the document is currently in timing mode.\n * Status is either\n * - 'active' if timings are enabled.\n * - 'pending' if timings are enabled but we can't get the data yet (as engine is blocked)\n * - 'disabled' if timings are disabled.\n */\n timing(): Promise;\n /**\n * Starts recording timing information for the document. Throws exception if timing is already\n * in progress or you don't have permission to start timing.\n */\n startTiming(): Promise;\n stopTiming(): Promise;\n /**\n * Starts the transfer of all attachments from the old attachment storage to the new one.\n */\n transferAllAttachments(): Promise;\n /**\n * Returns the status of the attachment transfer.\n */\n getAttachmentTransferStatus(): Promise;\n /**\n * Retries type of attachment storage used by the document.\n */\n getAttachmentStore(): Promise<{ type: AttachmentStore }>;\n /**\n * Sets the attachment storage used by the document.\n */\n setAttachmentStore(type: AttachmentStore): Promise;\n /**\n * Lists available external attachment stores. For now it contains at most one store.\n * If there is one store available it means that external storage is configured and can be used by this document.\n */\n getAttachmentStores(): Promise<{ stores: AttachmentStoreDesc[] }>;\n\n makeProposal(options?: {\n retracted?: boolean,\n }): Promise;\n getProposals(options?: {\n outgoing?: boolean\n }): Promise<{ proposals: Proposal[] }>;\n applyProposal(proposalId: number): Promise;\n\n applyUserActions(actions: UserAction[]): Promise;\n}\n\n// Operations that are supported by a doc worker.\nexport interface DocWorkerAPI {\n readonly url: string;\n importDocToWorkspace(uploadId: number, workspaceId: number, settings?: BrowserSettings): Promise;\n upload(material: UploadType, filename?: string): Promise;\n downloadDoc(docId: string, template?: boolean): Promise;\n copyDoc(docId: string, template?: boolean, name?: string): Promise;\n}\n\nexport class UserAPIImpl extends BaseAPI implements UserAPI {\n constructor(private _homeUrl: string, private _options: IOptions = {}) {\n super(_options);\n }\n\n public forRemoved(): UserAPI {\n const extraParameters = new Map([[\"showRemoved\", \"1\"]]);\n return new UserAPIImpl(this._homeUrl, { ...this._options, extraParameters });\n }\n\n public async getSessionActive(): Promise {\n return this.requestJson(`${this._url}/api/session/access/active`, { method: \"GET\" });\n }\n\n public async setSessionActive(email: string, org?: string): Promise {\n const body = JSON.stringify({ email, org });\n return this.requestJson(`${this._url}/api/session/access/active`, { method: \"POST\", body });\n }\n\n public async getSessionAll(): Promise<{ users: FullUser[], orgs: Organization[] }> {\n return this.requestJson(`${this._url}/api/session/access/all`, { method: \"GET\" });\n }\n\n public async getOrgs(merged: boolean = false): Promise {\n return this.requestJson(`${this._url}/api/orgs?merged=${merged ? 1 : 0}`, { method: \"GET\" });\n }\n\n public async getWorkspace(workspaceId: number): Promise {\n return this.requestJson(`${this._url}/api/workspaces/${workspaceId}`, { method: \"GET\" });\n }\n\n public async getOrg(orgId: number | string): Promise {\n return this.requestJson(`${this._url}/api/orgs/${orgId}`, { method: \"GET\" });\n }\n\n public async getOrgWorkspaces(orgId: number | string, includeSupport = true): Promise {\n return this.requestJson(`${this._url}/api/orgs/${orgId}/workspaces?includeSupport=${includeSupport ? 1 : 0}`,\n { method: \"GET\" });\n }\n\n public async getOrgUsageSummary(orgId: number | string): Promise {\n return this.requestJson(`${this._url}/api/orgs/${orgId}/usage`, { method: \"GET\" });\n }\n\n public async getTemplates(): Promise {\n return this.requestJson(`${this._url}/api/templates`, { method: \"GET\" });\n }\n\n public async getTemplate(docId: string): Promise {\n return this.requestJson(`${this._url}/api/templates/${docId}`, { method: \"GET\" });\n }\n\n public async getWidgets(): Promise {\n return await this.requestJson(`${this._url}/api/widgets`, { method: \"GET\" });\n }\n\n public async getDoc(docId: string): Promise {\n return this.requestJson(`${this._url}/api/docs/${docId}`, { method: \"GET\" });\n }\n\n public async newOrg(props: Partial): Promise {\n return this.requestJson(`${this._url}/api/orgs`, {\n method: \"POST\",\n body: JSON.stringify(props),\n });\n }\n\n public async newWorkspace(props: Partial, orgId: number | string): Promise {\n return this.requestJson(`${this._url}/api/orgs/${orgId}/workspaces`, {\n method: \"POST\",\n body: JSON.stringify(props),\n });\n }\n\n public async newDoc(props: Partial, workspaceId: number): Promise {\n return this.requestJson(`${this._url}/api/workspaces/${workspaceId}/docs`, {\n method: \"POST\",\n body: JSON.stringify(props),\n });\n }\n\n public async newUnsavedDoc(options: { timezone?: string } = {}): Promise {\n return this.requestJson(`${this._url}/api/docs`, {\n method: \"POST\",\n body: JSON.stringify(options),\n });\n }\n\n public async copyDoc(\n sourceDocumentId: string,\n workspaceId: number,\n options: CopyDocOptions,\n ): Promise {\n return this.requestJson(`${this._url}/api/docs`, {\n method: \"POST\",\n body: JSON.stringify({\n sourceDocumentId,\n workspaceId,\n ...options,\n }),\n });\n }\n\n public async renameOrg(orgId: number | string, name: string): Promise {\n await this.request(`${this._url}/api/orgs/${orgId}`, {\n method: \"PATCH\",\n body: JSON.stringify({ name }),\n });\n }\n\n public async renameWorkspace(workspaceId: number, name: string): Promise {\n await this.request(`${this._url}/api/workspaces/${workspaceId}`, {\n method: \"PATCH\",\n body: JSON.stringify({ name }),\n });\n }\n\n public async renameDoc(docId: string, name: string, { icon }: RenameDocOptions = {}): Promise {\n return this.updateDoc(docId, {\n name,\n ...(icon ? { options: { appearance: { icon } } } : undefined),\n });\n }\n\n public async updateOrg(orgId: number | string, props: Partial): Promise {\n await this.request(`${this._url}/api/orgs/${orgId}`, {\n method: \"PATCH\",\n body: JSON.stringify(props),\n });\n }\n\n public async updateDoc(docId: string, props: Partial): Promise {\n await this.request(`${this._url}/api/docs/${docId}`, {\n method: \"PATCH\",\n body: JSON.stringify(props),\n });\n }\n\n public async deleteOrg(orgId: number | string): Promise {\n await this.request(`${this._url}/api/orgs/${orgId}/force-delete`, { method: \"DELETE\" });\n }\n\n public async deleteWorkspace(workspaceId: number): Promise {\n await this.request(`${this._url}/api/workspaces/${workspaceId}`, { method: \"DELETE\" });\n }\n\n public async softDeleteWorkspace(workspaceId: number): Promise {\n await this.request(`${this._url}/api/workspaces/${workspaceId}/remove`, { method: \"POST\" });\n }\n\n public async undeleteWorkspace(workspaceId: number): Promise {\n await this.request(`${this._url}/api/workspaces/${workspaceId}/unremove`, { method: \"POST\" });\n }\n\n public async deleteDoc(docId: string): Promise {\n await this.request(`${this._url}/api/docs/${docId}`, { method: \"DELETE\" });\n }\n\n public async softDeleteDoc(docId: string): Promise {\n await this.request(`${this._url}/api/docs/${docId}/remove`, { method: \"POST\" });\n }\n\n public async undeleteDoc(docId: string): Promise {\n await this.request(`${this._url}/api/docs/${docId}/unremove`, { method: \"POST\" });\n }\n\n public async disableDoc(docId: string): Promise {\n await this.request(`${this._url}/api/docs/${docId}/disable`, { method: \"POST\" });\n }\n\n public async enableDoc(docId: string): Promise {\n await this.request(`${this._url}/api/docs/${docId}/enable`, { method: \"POST\" });\n }\n\n public async updateOrgPermissions(orgId: number | string, delta: PermissionDelta): Promise {\n await this.request(`${this._url}/api/orgs/${orgId}/access`, {\n method: \"PATCH\",\n body: JSON.stringify({ delta }),\n });\n }\n\n public async updateWorkspacePermissions(workspaceId: number, delta: PermissionDelta): Promise {\n await this.request(`${this._url}/api/workspaces/${workspaceId}/access`, {\n method: \"PATCH\",\n body: JSON.stringify({ delta }),\n });\n }\n\n public async updateDocPermissions(docId: string, delta: PermissionDelta): Promise {\n await this.request(`${this._url}/api/docs/${docId}/access`, {\n method: \"PATCH\",\n body: JSON.stringify({ delta }),\n });\n }\n\n public async getOrgAccess(orgId: number | string): Promise {\n return this.requestJson(`${this._url}/api/orgs/${orgId}/access`, { method: \"GET\" });\n }\n\n public async getWorkspaceAccess(workspaceId: number): Promise {\n return this.requestJson(`${this._url}/api/workspaces/${workspaceId}/access`, { method: \"GET\" });\n }\n\n public async getDocAccess(docId: string): Promise {\n return this.requestJson(`${this._url}/api/docs/${docId}/access`, { method: \"GET\" });\n }\n\n public async pinDoc(docId: string): Promise {\n await this.request(`${this._url}/api/docs/${docId}/pin`, {\n method: \"PATCH\",\n });\n }\n\n public async unpinDoc(docId: string): Promise {\n await this.request(`${this._url}/api/docs/${docId}/unpin`, {\n method: \"PATCH\",\n });\n }\n\n public async moveDoc(docId: string, workspaceId: number): Promise {\n await this.request(`${this._url}/api/docs/${docId}/move`, {\n method: \"PATCH\",\n body: JSON.stringify({ workspace: workspaceId }),\n });\n }\n\n public async getUserProfile(): Promise {\n return this.requestJson(`${this._url}/api/profile/user`);\n }\n\n public async updateUserName(name: string): Promise {\n await this.request(`${this._url}/api/profile/user/name`, {\n method: \"POST\",\n body: JSON.stringify({ name }),\n });\n }\n\n public async updateUserLocale(locale: string | null): Promise {\n await this.request(`${this._url}/api/profile/user/locale`, {\n method: \"POST\",\n body: JSON.stringify({ locale }),\n });\n }\n\n public async updateAllowGoogleLogin(allowGoogleLogin: boolean): Promise {\n await this.request(`${this._url}/api/profile/allowGoogleLogin`, {\n method: \"POST\",\n body: JSON.stringify({ allowGoogleLogin }),\n });\n }\n\n public async updateIsConsultant(userId: number, isConsultant: boolean): Promise {\n await this.request(`${this._url}/api/profile/isConsultant`, {\n method: \"POST\",\n body: JSON.stringify({ userId, isConsultant }),\n });\n }\n\n public async disableUser(userId: number): Promise {\n await this.request(`${this._url}/api/users/${userId}/disable`, {\n method: \"POST\",\n });\n }\n\n public async enableUser(userId: number): Promise {\n await this.request(`${this._url}/api/users/${userId}/enable`, {\n method: \"POST\",\n });\n }\n\n public async getWorker(key: string): Promise {\n const full = await this.getWorkerFull(key);\n return getPublicDocWorkerUrl(this._homeUrl, full);\n }\n\n public async getWorkerFull(key: string): Promise {\n const json = (await this.requestJson(`${this._url}/api/worker/${key}`, {\n method: \"GET\",\n credentials: \"include\",\n })) as PublicDocWorkerUrlInfo;\n return json;\n }\n\n public async getWorkerAPI(key: string): Promise {\n const docUrl = this._urlWithOrg(await this.getWorker(key));\n return new DocWorkerAPIImpl(docUrl, this._options);\n }\n\n public getBillingAPI(): BillingAPI {\n return new BillingAPIImpl(this._url, this._options);\n }\n\n public getDocAPI(docId: string): DocAPI {\n return new DocAPIImpl(this._url, docId, this._options);\n }\n\n public async fetchApiKey(): Promise {\n const resp = await this.request(`${this._url}/api/profile/apiKey`);\n return await resp.text();\n }\n\n public async createApiKey(): Promise {\n const res = await this.request(`${this._url}/api/profile/apiKey`, {\n method: \"POST\",\n });\n return await res.text();\n }\n\n public async deleteApiKey(): Promise {\n await this.request(`${this._url}/api/profile/apiKey`, {\n method: \"DELETE\",\n });\n }\n\n // This method is not strictly needed anymore, but is widely used by\n // tests so supporting as a handy shortcut for getDocAPI(docId).getRows(tableName)\n public async getTable(docId: string, tableName: string): Promise {\n return this.getDocAPI(docId).getRows(tableName);\n }\n\n public async applyUserActions(docId: string, actions: UserAction[]): Promise {\n return this.requestJson(`${this._url}/api/docs/${docId}/apply`, {\n method: \"POST\",\n body: JSON.stringify(actions),\n });\n }\n\n public async importUnsavedDoc(material: UploadType, options?: {\n filename?: string,\n timezone?: string,\n onUploadProgress?: (ev: AxiosProgressEvent) => void,\n }): Promise {\n options = options || {};\n const formData = this.newFormData();\n formData.append(\"upload\", material as any, options.filename);\n if (options.timezone) { formData.append(\"timezone\", options.timezone); }\n const resp = await this.requestAxios(`${this._url}/api/docs`, {\n method: \"POST\",\n data: formData,\n onUploadProgress: options.onUploadProgress,\n // On browser, it is important not to set Content-Type so that the browser takes care\n // of setting HTTP headers appropriately. Outside browser, requestAxios has logic\n // for setting the HTTP headers.\n headers: { ...this.defaultHeadersWithoutContentType() },\n });\n return resp.data;\n }\n\n public async deleteUser(userId: number, name: string) {\n await this.request(`${this._url}/api/users/${userId}`,\n { method: \"DELETE\",\n body: JSON.stringify({ name }) });\n }\n\n public async closeAccount(userId: number): Promise {\n return await this.requestJson(`${this._url}/api/doom/account?userid=` + userId, { method: \"DELETE\" });\n }\n\n public async closeOrg() {\n await this.request(`${this._url}/api/doom/org`, { method: \"DELETE\" });\n }\n\n public getBaseUrl(): string { return this._url; }\n\n // Recomputes the URL on every call to pick up changes in the URL when switching orgs.\n // (Feels inefficient, but probably doesn't matter, and it's simpler than the alternatives.)\n private get _url(): string {\n return this._urlWithOrg(this._homeUrl);\n }\n\n private _urlWithOrg(base: string): string {\n return isClient() ? addCurrentOrgToPath(base) : base.replace(/\\/$/, \"\");\n }\n}\n\nexport class DocWorkerAPIImpl extends BaseAPI implements DocWorkerAPI {\n constructor(public readonly url: string, _options: IOptions = {}) {\n super(_options);\n }\n\n public async importDocToWorkspace(uploadId: number, workspaceId: number, browserSettings?: BrowserSettings):\n Promise {\n return this.requestJson(`${this.url}/api/workspaces/${workspaceId}/import`, {\n method: \"POST\",\n body: JSON.stringify({ uploadId, browserSettings }),\n });\n }\n\n public async upload(material: UploadType, filename?: string): Promise {\n const formData = this.newFormData();\n formData.append(\"upload\", material as any, filename);\n const json = await this.requestJson(`${this.url}/uploads`, {\n // On browser, it is important not to set Content-Type so that the browser takes care\n // of setting HTTP headers appropriately. Outside of browser, node-fetch also appears\n // to take care of this - https://github.github.io/fetch/#request-body\n headers: { ...this.defaultHeadersWithoutContentType() },\n method: \"POST\",\n body: formData,\n });\n return json.uploadId;\n }\n\n public async downloadDoc(docId: string, template: boolean = false): Promise {\n const extra = template ? \"?template=1\" : \"\";\n const result = await this.request(`${this.url}/api/docs/${docId}/download${extra}`, {\n method: \"GET\",\n });\n if (!result.ok) { throw new Error(await result.text()); }\n return result;\n }\n\n public async copyDoc(docId: string, template: boolean = false, name?: string): Promise {\n const url = new URL(`${this.url}/copy?doc=${docId}`);\n if (template) {\n url.searchParams.append(\"template\", \"1\");\n }\n if (name) {\n url.searchParams.append(\"name\", name);\n }\n const json = await this.requestJson(url.href, {\n method: \"POST\",\n });\n return json.uploadId;\n }\n}\n\nexport class DocAPIImpl extends BaseAPI implements DocAPI {\n private _url: string;\n\n constructor(url: string, public readonly docId: string, options: IOptions = {}) {\n super(options);\n this._url = `${url}/api/docs/${docId}`;\n }\n\n public getBaseUrl(): string { return this._url; }\n\n public async getTables(options?: GetTablesParams): Promise {\n const url = new URL(`${this._url}/tables`);\n if (options?.expand) {\n url.searchParams.set(\"expand\", options.expand.join(\",\"));\n }\n return this.requestJson(url.href);\n }\n\n public async getRows(tableId: string, options?: GetRowsParams): Promise {\n return this._getRecords(tableId, \"data\", options);\n }\n\n public async getRecords(tableId: string, options?: GetRowsParams): Promise {\n const response: TableRecordValues = await this._getRecords(tableId, \"records\", options);\n return response.records;\n }\n\n public async sql(sql: string, args?: any[]): Promise {\n return this.requestJson(`${this._url}/sql`, {\n body: JSON.stringify({\n sql,\n ...(args ? { args } : {}),\n }),\n method: \"POST\",\n });\n }\n\n public async updateRows(tableId: string, changes: TableColValues): Promise {\n return this.requestJson(`${this._url}/tables/${tableId}/data`, {\n body: JSON.stringify(changes),\n method: \"PATCH\",\n });\n }\n\n /**\n * Adds rows to a table.\n *\n * Example:\n * ```typescript\n * const newRowIds = await docApi.addRows(\"tableId\", {\n * \"Column1\": [value1, value2],\n * \"Column2\": [value3, value4],\n * });\n * ```\n * @param tableId Table ID to add rows to.\n * @param additions JSON object with column values for the new rows.\n * @returns Array of new row IDs.\n */\n public async addRows(tableId: string, additions: BulkColValues): Promise {\n return this.requestJson(`${this._url}/tables/${tableId}/data`, {\n body: JSON.stringify(additions),\n method: \"POST\",\n });\n }\n\n public async removeRows(tableId: string, removals: number[]): Promise {\n return this.requestJson(`${this._url}/tables/${tableId}/records/delete`, {\n body: JSON.stringify(removals),\n method: \"POST\",\n });\n }\n\n public async fork(): Promise {\n return this.requestJson(`${this._url}/fork`, {\n method: \"POST\",\n });\n }\n\n public async replace(source: DocReplacementOptions): Promise {\n return this.requestJson(`${this._url}/replace`, {\n body: JSON.stringify(source),\n method: \"POST\",\n });\n }\n\n public async getSnapshots(raw?: boolean): Promise {\n return this.requestJson(`${this._url}/snapshots?raw=${raw}`);\n }\n\n public async removeSnapshots(snapshotIds: string[] | \"unlisted\" | \"past\") {\n const body = typeof snapshotIds === \"string\" ? { select: snapshotIds } : { snapshotIds };\n return await this.requestJson(`${this._url}/snapshots/remove`, {\n method: \"POST\",\n body: JSON.stringify(body),\n });\n }\n\n public async getStates(): Promise {\n return this.requestJson(`${this._url}/states`);\n }\n\n public async getUsersForViewAs(): Promise {\n return this.requestJson(`${this._url}/usersForViewAs`);\n }\n\n public async getWebhooks(): Promise {\n return this.requestJson(`${this._url}/webhooks`);\n }\n\n public async addWebhook(webhook: WebhookSubscribe & { tableId: string }): Promise<{ webhookId: string }> {\n const { tableId } = webhook;\n return this.requestJson(`${this._url}/tables/${tableId}/_subscribe`, {\n method: \"POST\",\n body: JSON.stringify(\n omitBy(webhook, (val, key) => key === \"tableId\" || val === null)),\n });\n }\n\n public async updateWebhook(webhook: WebhookUpdate): Promise {\n return this.requestJson(`${this._url}/webhooks/${webhook.id}`, {\n method: \"PATCH\",\n body: JSON.stringify(webhook.fields),\n });\n }\n\n public removeWebhook(webhookId: string, tableId: string) {\n // unsubscribeKey is not required for owners\n const unsubscribeKey = \"\";\n return this.requestJson(`${this._url}/tables/${tableId}/_unsubscribe`, {\n method: \"POST\",\n body: JSON.stringify({ webhookId, unsubscribeKey }),\n });\n }\n\n public async flushWebhooks(): Promise {\n await this.request(`${this._url}/webhooks/queue`, {\n method: \"DELETE\",\n });\n }\n\n public async flushWebhook(id: string): Promise {\n await this.request(`${this._url}/webhooks/queue/${id}`, {\n method: \"DELETE\",\n });\n }\n\n public async forceReload(): Promise {\n await this.request(`${this._url}/force-reload`, {\n method: \"POST\",\n });\n }\n\n public async recover(recoveryMode: boolean): Promise {\n await this.request(`${this._url}/recover`, {\n body: JSON.stringify({ recoveryMode }),\n method: \"POST\",\n });\n }\n\n public async compareDoc(\n remoteDocId: string,\n options: {\n detail?: boolean;\n maxRows?: number | null;\n } = {},\n ): Promise {\n const { detail, maxRows } = options;\n const url = new URL(`${this._url}/compare/${remoteDocId}`);\n if (detail) {\n url.searchParams.set(\"detail\", \"true\");\n }\n if (maxRows !== undefined) {\n url.searchParams.set(\"maxRows\", String(maxRows));\n }\n return this.requestJson(url.href);\n }\n\n public async copyDoc(workspaceId: number, options: CopyDocOptions): Promise {\n const { documentName, asTemplate } = options;\n return this.requestJson(`${this._url}/copy`, {\n body: JSON.stringify({ workspaceId, documentName, asTemplate }),\n method: \"POST\",\n });\n }\n\n public async compareVersion(leftHash: string, rightHash: string): Promise {\n const url = new URL(`${this._url}/compare`);\n url.searchParams.append(\"left\", leftHash);\n url.searchParams.append(\"right\", rightHash);\n return this.requestJson(url.href);\n }\n\n public getDownloadUrl({ template, removeHistory}: { template: boolean, removeHistory: boolean }): string {\n return this._url + `/download?template=${template}&nohistory=${removeHistory}`;\n }\n\n public getDownloadXlsxUrl(params: DownloadDocParams) {\n return this._url + \"/download/xlsx?\" + encodeQueryParams({ ...params });\n }\n\n public getDownloadCsvUrl(params: DownloadDocParams) {\n // We spread `params` to work around TypeScript being overly cautious.\n return this._url + \"/download/csv?\" + encodeQueryParams({ ...params });\n }\n\n public getDownloadTsvUrl(params: DownloadDocParams) {\n return this._url + \"/download/tsv?\" + encodeQueryParams({ ...params });\n }\n\n public getDownloadDsvUrl(params: DownloadDocParams) {\n return this._url + \"/download/dsv?\" + encodeQueryParams({ ...params });\n }\n\n public getDownloadTableSchemaUrl(params: DownloadDocParams) {\n // We spread `params` to work around TypeScript being overly cautious.\n return this._url + \"/download/table-schema?\" + encodeQueryParams({ ...params });\n }\n\n public getDownloadAttachmentsArchiveUrl(params: AttachmentsArchiveParams): string {\n return this._url + \"/attachments/archive?\" + encodeQueryParams({ ...params });\n }\n\n public async sendToDrive(code: string, title: string): Promise<{ url: string }> {\n const url = new URL(`${this._url}/send-to-drive`);\n url.searchParams.append(\"title\", title);\n url.searchParams.append(\"code\", code);\n return this.requestJson(url.href);\n }\n\n public async uploadAttachment(value: string | Blob, filename?: string): Promise {\n const formData = this.newFormData();\n formData.append(\"upload\", value, filename);\n const response = await this.requestAxios(`${this._url}/attachments`, {\n method: \"POST\",\n data: formData,\n // On browser, it is important not to set Content-Type so that the browser takes care\n // of setting HTTP headers appropriately. Outside browser, requestAxios has logic\n // for setting the HTTP headers.\n headers: { ...this.defaultHeadersWithoutContentType() },\n });\n return response.data[0];\n }\n\n public async uploadAttachmentArchive(archive: string | Blob, filename?: string): Promise {\n const formData = this.newFormData();\n formData.append(\"upload\", archive, filename);\n const response = await this.requestAxios(`${this._url}/attachments/archive`, {\n method: \"POST\",\n data: formData,\n // On the browser, Content-Type shouldn't be set as it prevents the browser from setting\n // Content-Type with the correct boundary expression to delimit form fields.\n // https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest_API/Using_FormData_Objects#sending_files_using_a_formdata_object\n // Therefore we omit Content-Type, and allow Axios to handle it as it sees fit - which works\n // correctly in the browser and in Node.\n headers: { ...this.defaultHeadersWithoutContentType() },\n });\n return response.data;\n }\n\n public async getAssistance(\n params: AssistanceRequest,\n ): Promise {\n return await this.requestJson(`${this._url}/assistant`, {\n method: \"POST\",\n body: JSON.stringify(params),\n });\n }\n\n public async timing(): Promise {\n return this.requestJson(`${this._url}/timing`);\n }\n\n public async startTiming(): Promise {\n await this.request(`${this._url}/timing/start`, { method: \"POST\" });\n }\n\n public async stopTiming(): Promise {\n return await this.requestJson(`${this._url}/timing/stop`, { method: \"POST\" });\n }\n\n public async transferAllAttachments(): Promise {\n await this.request(`${this._url}/attachments/transferAll`, { method: \"POST\" });\n }\n\n public async getAttachmentTransferStatus(): Promise {\n return this.requestJson(`${this._url}/attachments/transferStatus`);\n }\n\n public async getAttachmentStore(): Promise<{ type: AttachmentStore }> {\n return this.requestJson(`${this._url}/attachments/store`);\n }\n\n public async getAttachmentStores(): Promise<{ stores: AttachmentStoreDesc[] }> {\n return this.requestJson(`${this._url}/attachments/stores`);\n }\n\n public async setAttachmentStore(type: AttachmentStore): Promise {\n await this.request(`${this._url}/attachments/store`, {\n method: \"POST\",\n body: JSON.stringify({ type }),\n });\n }\n\n public async makeProposal(options?: {\n retracted?: boolean,\n }) {\n return this.requestJson(`${this._url}/propose`, {\n method: \"POST\",\n body: JSON.stringify(options || {}),\n });\n }\n\n public async applyProposal(proposalId: number) {\n return this.requestJson(`${this._url}/proposals/${proposalId}/apply`, {\n method: \"POST\",\n });\n }\n\n public async applyUserActions(actions: UserAction[]): Promise {\n return this.requestJson(`${this._url}/apply`, {\n method: \"POST\",\n body: JSON.stringify(actions),\n });\n }\n\n public async getProposals(options?: {\n outgoing?: boolean,\n }) {\n const result = await this.requestJson(`${this._url}/proposals?outgoing=${Boolean(options?.outgoing)}`, {\n method: \"GET\",\n });\n return result;\n }\n\n private _getRecords(tableId: string, endpoint: \"data\" | \"records\", options?: GetRowsParams): Promise {\n const url = new URL(`${this._url}/tables/${tableId}/${endpoint}`);\n if (options?.filters) {\n url.searchParams.append(\"filter\", JSON.stringify(options.filters));\n }\n if (options?.immediate) {\n url.searchParams.append(\"immediate\", \"true\");\n }\n return this.requestJson(url.href);\n }\n}\n\nexport interface AttachmentTransferStatus {\n status: {\n pendingTransferCount: number;\n isRunning: boolean;\n\n // Count of failures and successes since starting a\n // transfer of all files.\n failures: number;\n successes: number;\n };\n locationSummary: DocAttachmentsLocation;\n}\n\n/**\n * Represents information to build public doc worker url.\n *\n * Structure that may contain either **exclusively**:\n * - a selfPrefix when no pool of doc worker exist.\n * - a public doc worker url otherwise.\n */\nexport type PublicDocWorkerUrlInfo = {\n selfPrefix: string;\n docWorkerUrl: null;\n docWorkerId: null;\n} | {\n selfPrefix: null;\n docWorkerUrl: string;\n docWorkerId: string;\n};\n\nexport function getUrlFromPrefix(homeUrl: string, prefix: string) {\n const url = new URL(homeUrl);\n url.pathname = prefix + url.pathname;\n return url.href;\n}\n\n/**\n * Get a docWorkerUrl from information returned from backend. When the backend\n * is fully configured, and there is a pool of workers, this is straightforward,\n * just return the docWorkerUrl reported by the backend. For single-instance\n * installs, the backend returns a null docWorkerUrl, and a client can simply\n * use the homeUrl of the backend, with extra path prefix information\n * given by selfPrefix. At the time of writing, the selfPrefix contains a\n * doc-worker id, and a tag for the codebase (used in consistency checks).\n *\n * @param {string} homeUrl\n * @param {string} docWorkerInfo The information to build the public doc worker url\n * (result of the call to /api/worker/:docId)\n */\nexport function getPublicDocWorkerUrl(homeUrl: string, docWorkerInfo: PublicDocWorkerUrlInfo) {\n return docWorkerInfo.selfPrefix !== null ?\n getUrlFromPrefix(homeUrl, docWorkerInfo.selfPrefix) :\n docWorkerInfo.docWorkerUrl;\n}\n" }, { "path": "app/common/UserConfig.ts", "content": "/*\n * Interface for the user's config found in config.json.\n */\nexport interface UserConfig {\n docListSortBy?: string;\n docListSortDir?: number;\n features?: ISupportedFeatures;\n\n /*\n * The host serving the untrusted content: on dev environment could be\n * \"http://getgrist.localtest.me\". Port is added at runtime and should not be included.\n */\n untrustedContentOrigin?: string;\n}\n\nexport interface ISupportedFeatures {\n signin?: boolean;\n sharing?: boolean;\n proxy?: boolean; // If true, Grist will accept login information via http headers\n // X-Forwarded-User and X-Forwarded-Email. Set to true only if\n // Grist is behind a reverse proxy that is managing those headers,\n // otherwise they could be spoofed.\n formulaBar?: boolean;\n\n // Plugin views, and REPL all need work, but are exposed here to allow existing\n // tests to continue running. These only affect client-side code.\n customViewPlugin?: boolean;\n}\n" }, { "path": "app/common/ValueConverter.ts", "content": "import { TableDataActionSet } from \"app/common/DocActions\";\nimport { DocData } from \"app/common/DocData\";\nimport * as gristTypes from \"app/common/gristTypes\";\nimport { isList } from \"app/common/gristTypes\";\nimport { BaseFormatter, createFullFormatterFromDocData } from \"app/common/ValueFormatter\";\nimport {\n createParserOrFormatterArgumentsRaw,\n createParserRaw,\n ReferenceListParser,\n ReferenceParser,\n ValueParser,\n} from \"app/common/ValueParser\";\nimport { CellValue, GristObjCode } from \"app/plugin/GristData\";\n\n/**\n * Base class for converting values from one type to another with the convert() method.\n * Has a formatter for the source column\n * and a parser for the destination column.\n *\n * The default convert() is for non-list destination types, so if the source value\n * is a list it only converts nicely if the list contains exactly one element.\n */\nexport class ValueConverter {\n private _isTargetText: boolean = [\"Text\", \"Choice\"].includes(this.parser.type);\n\n constructor(public formatter: BaseFormatter, public parser: ValueParser) {\n }\n\n public convert(value: any): any {\n if (isList(value)) {\n if (value.length === 1) {\n // Empty list: ['L']\n return null;\n } else if (value.length > 2 || this._isTargetText) {\n // List with multiple values, or the target type is text.\n // Since we're converting to just one value,\n // format the whole thing as text, which is an error for most types.\n return this.formatter.formatAny(value);\n } else {\n // Singleton list: ['L', value]\n // Convert just that one value.\n value = value[1];\n }\n }\n return this.convertInner(value);\n }\n\n protected convertInner(value: any): any {\n const formatted = this.formatter.formatAny(value);\n return this.parser.cleanParse(formatted);\n }\n}\n\n/**\n * Base class for converting to a list type (Reference List or Choice List).\n *\n * Wraps single values in a list, and converts lists elementwise.\n */\nclass ListConverter extends ValueConverter {\n // Don't parse strings like \"Smith, John\" which may look like lists but represent a single choice.\n // TODO this works when the source is a Choice column, but not when it's a Reference to a Choice column.\n // But the guessed choices are also broken in that case.\n private _choices = new Set((this.formatter.widgetOpts as any).choices || []);\n\n public convert(value: any): any {\n if (typeof value === \"string\" && !this._choices.has(value)) {\n // Parse CSV/JSON\n return this.parser.cleanParse(value);\n }\n const values = isList(value) ? value.slice(1) : [value];\n if (!values.length || value == null) {\n return null;\n }\n return this.handleValues(value, values.map(v => this.convertInner(v)));\n }\n\n protected handleValues(originalValue: any, values: any[]) {\n return [\"L\", ...values];\n }\n}\n\nclass ChoiceListConverter extends ListConverter {\n /**\n * Convert each source value to a 'Choice'\n */\n protected convertInner(value: any): any {\n return this.formatter.formatAny(value);\n }\n}\n\nclass ReferenceListConverter extends ListConverter {\n private _innerConverter = new ReferenceConverter(\n this.formatter,\n new ReferenceParser(\"Ref\", this.parser.widgetOpts, this.parser.docSettings),\n );\n\n constructor(public formatter: BaseFormatter, public parser: ReferenceListParser) {\n super(formatter, parser);\n // Prevent the parser from looking up reference values in the frontend.\n // Leave it to the data engine which has a much more efficient algorithm for long lists of values.\n delete parser.tableData;\n }\n\n public handleValues(originalValue: any, values: any[]): any {\n const result = [];\n let lookupColumn: string = \"\";\n const raw = this.formatter.formatAny(originalValue); // AltText if the reference lookup fails\n for (const value of values) {\n if (typeof value === \"string\") {\n // Failed to parse one of the references, so return a raw string for the whole thing\n return raw;\n } else {\n // value is a lookup tuple: ['l', value, options]\n result.push(value[1]);\n lookupColumn = value[2].column;\n }\n }\n return [\"l\", result, { column: lookupColumn, raw }];\n }\n\n /**\n * Convert each source value to a 'Reference'\n */\n protected convertInner(value: any): any {\n return this._innerConverter.convert(value);\n }\n}\n\nclass ReferenceConverter extends ValueConverter {\n private _innerConverter: ValueConverter = createConverter(this.formatter, this.parser.visibleColParser);\n\n constructor(public formatter: BaseFormatter, public parser: ReferenceParser) {\n super(formatter, parser);\n // Prevent the parser from looking up reference values in the frontend.\n // Leave it to the data engine which has a much more efficient algorithm for long lists of values.\n delete parser.tableData;\n }\n\n protected convertInner(value: any): any {\n // Convert to the type of the visible column.\n const converted = this._innerConverter.convert(value);\n return this.parser.lookup(converted, this.formatter.formatAny(value));\n }\n}\n\nclass NumericConverter extends ValueConverter {\n protected convertInner(value: any): any {\n if (typeof value === \"boolean\") {\n return value ? 1 : 0;\n }\n return super.convertInner(value);\n }\n}\n\nclass DateConverter extends ValueConverter {\n private _sourceType = gristTypes.extractInfoFromColType(this.formatter.type);\n\n protected convertInner(value: any): any {\n // When converting Date->DateTime, DateTime->Date, or between DateTime timezones,\n // it's important to send an encoded Date/DateTime object rather than just a timestamp number\n // so that the data engine knows what to do in do_convert, especially regarding timezones.\n // If the source column is a Reference to a Date/DateTime then `value` is already\n // an encoded object from the display column which has type Any.\n value = gristTypes.reencodeAsTypedCellValue(value, this._sourceType);\n if (Array.isArray(value) && (\n value[0] === GristObjCode.Date ||\n value[0] === GristObjCode.DateTime\n )) {\n return value;\n }\n return super.convertInner(value);\n }\n}\n\nexport const valueConverterClasses: { [type: string]: typeof ValueConverter } = {\n Date: DateConverter,\n DateTime: DateConverter,\n ChoiceList: ChoiceListConverter,\n Ref: ReferenceConverter,\n RefList: ReferenceListConverter,\n Numeric: NumericConverter,\n Int: NumericConverter,\n};\n\nexport function createConverter(formatter: BaseFormatter, parser: ValueParser) {\n const cls = valueConverterClasses[gristTypes.extractTypeFromColType(parser.type)] || ValueConverter;\n return new cls(formatter, parser);\n}\n\n/**\n * Used by the ConvertFromColumn user action in the data engine.\n * The higher order function separates docData (passed by ActiveDoc)\n * from the arguments passed to call_external in Python.\n */\nexport function convertFromColumn(\n metaTables: TableDataActionSet,\n sourceColRef: number,\n type: string,\n widgetOpts: string,\n visibleColRef: number,\n values: readonly CellValue[],\n displayColValues?: readonly CellValue[],\n): CellValue[] {\n const docData = new DocData(\n (_tableId) => { throw new Error(\"Unexpected DocData fetch\"); },\n metaTables,\n );\n\n const formatter = createFullFormatterFromDocData(docData, sourceColRef);\n const parser = createParserRaw(\n ...createParserOrFormatterArgumentsRaw(docData, type, widgetOpts, visibleColRef),\n );\n const converter = createConverter(formatter, parser);\n return convertValues(converter, values, displayColValues || values);\n}\n\nexport function convertValues(\n converter: ValueConverter,\n // Raw values from the actual column, e.g. row IDs for reference columns\n values: readonly CellValue[],\n // Values from the display column, which is the same as the raw values for non-referencing columns.\n // In almost all cases these are the values that actually matter and get converted.\n displayColValues: readonly CellValue[],\n): CellValue[] {\n // Converting Ref <-> RefList without changing the target table is a special case - see prepTransformColInfo.\n // In this case we deal with the actual row IDs stored in the real column,\n // whereas in all other cases we use display column values.\n const sourceType = gristTypes.extractInfoFromColType(converter.formatter.type);\n const targetType = gristTypes.extractInfoFromColType(converter.parser.type);\n const refToRefList = (\n sourceType.type === \"Ref\" &&\n targetType.type === \"RefList\" &&\n sourceType.tableId === targetType.tableId\n );\n const refListToRef = (\n sourceType.type === \"RefList\" &&\n targetType.type === \"Ref\" &&\n sourceType.tableId === targetType.tableId\n );\n\n return displayColValues.map((displayVal, i) => {\n const actualValue = values[i];\n\n if (refToRefList && typeof actualValue === \"number\") {\n if (actualValue === 0) {\n return null;\n } else {\n return [\"L\", actualValue];\n }\n } else if (refListToRef && isList(actualValue)) {\n if (actualValue.length === 1) {\n // Empty list: ['L']\n return 0;\n } else if (actualValue.length === 2) {\n // Singleton list: ['L', rowId]\n return actualValue[1];\n }\n }\n\n return converter.convert(displayVal);\n });\n}\n" }, { "path": "app/common/ValueFormatter.ts", "content": "import { csvEncodeRow } from \"app/common/csvFormat\";\nimport { CellValue } from \"app/common/DocActions\";\nimport { DocData } from \"app/common/DocData\";\nimport { DocumentSettings } from \"app/common/DocumentSettings\";\nimport * as gristTypes from \"app/common/gristTypes\";\nimport { getReferencedTableId, isList } from \"app/common/gristTypes\";\nimport * as gutil from \"app/common/gutil\";\nimport { isHiddenTable } from \"app/common/isHiddenTable\";\nimport { buildNumberFormat, NumberFormatOptions } from \"app/common/NumberFormat\";\nimport { createParserOrFormatterArguments, ReferenceParsingOptions } from \"app/common/ValueParser\";\nimport { GristObjCode } from \"app/plugin/GristData\";\nimport { decodeObject, GristDateTime } from \"app/plugin/objtypes\";\n\nimport isPlainObject from \"lodash/isPlainObject\";\nimport moment from \"moment-timezone\";\n\nexport { PENDING_DATA_PLACEHOLDER } from \"app/plugin/objtypes\";\n\nexport interface FormatOptions {\n [option: string]: any;\n}\n\n/**\n * Formats a value of any type generically (with no type-specific options).\n */\nexport function formatUnknown(value: CellValue): string {\n return formatDecoded(decodeObject(value));\n}\n\n/**\n * Returns true if the array contains other arrays or structured objects,\n * indicating that the list should be formatted like JSON rather than CSV.\n */\nfunction hasNestedObjects(value: any[]) {\n return value.some(v => typeof v === \"object\" && v && (Array.isArray(v) || isPlainObject(v)));\n}\n\n/**\n * Formats a decoded Grist value for displaying it. For top-level values, formats them the way we\n * like to see them in a cell or in, say, CSV export.\n * For top-level lists containing only simple values like strings and dates, formats them as a CSV row.\n * Nested lists and objects are formatted slightly differently, with quoted strings and ISO format for dates.\n */\nexport function formatDecoded(value: unknown, isTopLevel: boolean = true): string {\n if (typeof value === \"object\" && value) {\n if (Array.isArray(value)) {\n if (!isTopLevel || hasNestedObjects(value)) {\n return \"[\" + value.map(v => formatDecoded(v, false)).join(\", \") + \"]\";\n } else {\n return csvEncodeRow(value.map(v => formatDecoded(v, true)), { prettier: true });\n }\n } else if (isPlainObject(value)) {\n const obj: any = value;\n const items = Object.keys(obj).map(k => `${JSON.stringify(k)}: ${formatDecoded(obj[k], false)}`);\n return \"{\" + items.join(\", \") + \"}\";\n } else if (isTopLevel && value instanceof GristDateTime) {\n return moment(value).tz(value.timezone).format(\"YYYY-MM-DD HH:mm:ssZ\");\n }\n return String(value);\n }\n if (isTopLevel) {\n return (value == null ? \"\" : String(value));\n }\n return JSON.stringify(value);\n}\n\nexport type IsRightTypeFunc = (value: CellValue) => boolean;\n\nexport class BaseFormatter {\n protected isRightType: IsRightTypeFunc;\n\n constructor(public type: string, public widgetOpts: FormatOptions, public docSettings: DocumentSettings) {\n this.isRightType = gristTypes.isRightType(gristTypes.extractTypeFromColType(type)) ||\n gristTypes.isRightType(\"Any\")!;\n }\n\n /**\n * Formats using this.format() if a value is of the right type for this formatter, or using\n * AnyFormatter otherwise. This method the recommended API. There is no need to override it.\n */\n public formatAny(value: any, translate?: (val: string) => string): string {\n return this.isRightType(value) ? this.format(value, translate) : formatUnknown(value);\n }\n\n /**\n * Formats a value that matches the type of this formatter. This should be overridden by derived\n * classes to handle values in formatter-specific ways.\n */\n protected format(value: any, _translate?: (val: string) => string): string {\n return String(value);\n }\n}\n\nexport class BoolFormatter extends BaseFormatter {\n public format(value: boolean | 0 | 1, translate?: (val: string) => string): string {\n if (typeof value === \"boolean\" && translate) {\n return translate(String(value));\n }\n return super.format(value, translate);\n }\n}\n\nclass AnyFormatter extends BaseFormatter {\n public format(value: any): string {\n return formatUnknown(value);\n }\n}\n\nexport class NumericFormatter extends BaseFormatter {\n private _numFormat: Intl.NumberFormat;\n private _formatter: (val: number) => string;\n\n constructor(type: string, options: NumberFormatOptions, docSettings: DocumentSettings) {\n super(type, options, docSettings);\n this._numFormat = buildNumberFormat(options, docSettings);\n this._formatter = (options.numSign === \"parens\") ? this._formatParens : this._formatPlain;\n }\n\n public format(value: any): string {\n return value === null ? \"\" : this._formatter(value);\n }\n\n public _formatPlain(value: number): string {\n return this._numFormat.format(value);\n }\n\n public _formatParens(value: number): string {\n // Surround positive numbers with spaces to align them visually to parenthesized numbers.\n return (value >= 0) ?\n ` ${this._numFormat.format(value)} ` :\n `(${this._numFormat.format(-value)})`;\n }\n}\n\nclass IntFormatter extends NumericFormatter {\n constructor(type: string, opts: FormatOptions, docSettings: DocumentSettings) {\n super(type, { decimals: 0, ...opts }, docSettings);\n }\n}\n\nexport interface DateFormatOptions {\n dateFormat?: string;\n}\n\nclass DateFormatter extends BaseFormatter {\n protected _dateTimeFormat: string;\n private _timezone: string;\n\n constructor(type: string, widgetOpts: DateFormatOptions, docSettings: DocumentSettings, timezone: string = \"UTC\") {\n super(type, widgetOpts, docSettings);\n // Allow encoded dates/datetimes ([d, number] or [D, number, timezone])\n // which are found in formula columns of type Any,\n // particularly reference display columns which are formatted here according to the visible column\n // which will have the correct column type and options.\n // Since these encoded objects are not expected in a Date/Datetime column and require\n // being handled differently from just a number,\n // we don't change `gristTypes.isRightType` which is used elsewhere.\n this.isRightType = (value: any) => (\n value === null ||\n typeof value === \"number\" ||\n Array.isArray(value) && (\n value[0] === GristObjCode.Date ||\n value[0] === GristObjCode.DateTime\n )\n );\n this._dateTimeFormat = widgetOpts.dateFormat || \"YYYY-MM-DD\";\n this._timezone = timezone;\n }\n\n public format(value: any): string {\n if (value === null) {\n return \"\";\n }\n\n // For a DateTime object in an Any column, use the provided timezone (`value[2]`)\n // Otherwise use the timezone configured for a DateTime column.\n let timezone = this._timezone;\n if (Array.isArray(value)) {\n timezone = value[2] || timezone;\n value = value[1];\n }\n // Now `value` is a number\n\n const time = moment.tz(value * 1000, timezone);\n return time.format(this._dateTimeFormat);\n }\n}\n\nexport interface DateTimeFormatOptions extends DateFormatOptions {\n timeFormat?: string;\n}\n\nclass DateTimeFormatter extends DateFormatter {\n constructor(type: string, widgetOpts: DateTimeFormatOptions, docSettings: DocumentSettings) {\n const timezone = gutil.removePrefix(type, \"DateTime:\") || \"\";\n // Pass up the original widgetOpts. It's helpful to have them available; e.g. ExcelFormatter\n // takes options from an initialized ValueFormatter.\n super(type, widgetOpts, docSettings, timezone);\n const timeFormat = widgetOpts.timeFormat === undefined ? \"h:mma\" : widgetOpts.timeFormat;\n this._dateTimeFormat = (widgetOpts.dateFormat || \"YYYY-MM-DD\") + \" \" + timeFormat;\n }\n}\n\nclass RowIdFormatter extends BaseFormatter {\n public widgetOpts: { tableId: string };\n\n public format(value: number): string {\n return value > 0 ? `${this.widgetOpts.tableId}[${value}]` : \"\";\n }\n}\n\ninterface ReferenceFormatOptions {\n visibleColFormatter?: BaseFormatter;\n}\n\nclass ReferenceFormatter extends BaseFormatter {\n public widgetOpts: ReferenceFormatOptions;\n protected visibleColFormatter: BaseFormatter;\n\n constructor(type: string, widgetOpts: ReferenceFormatOptions, docSettings: DocumentSettings) {\n super(type, widgetOpts, docSettings);\n // widgetOpts.visibleColFormatter shouldn't be undefined, but it can be if a referencing column\n // is displaying another referencing column, which is partially prohibited in the UI but still possible.\n this.visibleColFormatter = widgetOpts.visibleColFormatter ||\n createFormatter(\"Id\", { tableId: getReferencedTableId(type) }, docSettings);\n }\n\n public formatAny(value: any): string {\n /*\n An invalid value in a referencing column is saved as a string and becomes AltText in the data engine.\n Then the display column formula (e.g. $person.first_name) raises an InvalidTypedValue trying to access\n an attribute of that AltText.\n This would normally lead to the formatter displaying `#Invalid Ref[List]: ` before the string value.\n That's inconsistent with how the cell is displayed (just the string value in pink)\n and with how invalid values in other columns are formatted (just the string).\n It's just a result of the formatter receiving a value from the display column, not the actual column.\n It's also likely to inconvenience users trying to import/migrate/convert data.\n So we suppress the error here and just show the text.\n It's still technically possible for the column to display an actual InvalidTypedValue exception from a formula\n and this will suppress that too, but this is unlikely and seems worth it.\n */\n if (\n Array.isArray(value) &&\n value[0] === GristObjCode.Exception &&\n value[1] === \"InvalidTypedValue\" &&\n value[2]?.startsWith?.(\"Ref\")\n ) {\n return value[3];\n }\n return this.formatNotInvalidRef(value);\n }\n\n protected formatNotInvalidRef(value: any) {\n return this.visibleColFormatter.formatAny(value);\n }\n}\n\nclass ReferenceListFormatter extends ReferenceFormatter {\n protected formatNotInvalidRef(value: any): string {\n // Part of this repeats the logic in BaseFormatter.formatAny which is overridden in ReferenceFormatter\n // It also ensures that complex lists (e.g. if this RefList is displaying a ChoiceList)\n // are formatted as JSON instead of CSV.\n if (!isList(value) || hasNestedObjects(decodeObject(value) as CellValue[])) {\n return formatUnknown(value);\n }\n // In the most common case, lists of simple objects like strings or dates\n // are formatted like a CSV.\n // This is similar to formatUnknown except the inner values are\n // formatted according to the visible column options.\n const formattedValues = value.slice(1).map(v => super.formatNotInvalidRef(v));\n return csvEncodeRow(formattedValues, { prettier: true });\n }\n}\n\nconst formatters: { [name: string]: typeof BaseFormatter } = {\n Numeric: NumericFormatter,\n Int: IntFormatter,\n Bool: BoolFormatter,\n Date: DateFormatter,\n DateTime: DateTimeFormatter,\n Ref: ReferenceFormatter,\n RefList: ReferenceListFormatter,\n Id: RowIdFormatter,\n // We don't list anything that maps to AnyFormatter, since that's the default.\n};\n\n/**\n * Takes column type, widget options and document settings, and returns a constructor\n * with a format function that can properly convert a value passed to it into the\n * right format for that column.\n */\nexport function createFormatter(type: string, widgetOpts: FormatOptions, docSettings: DocumentSettings): BaseFormatter {\n const ctor = formatters[gristTypes.extractTypeFromColType(type)] || AnyFormatter;\n return new ctor(type, widgetOpts, docSettings);\n}\n\nexport interface FullFormatterArgs {\n docData: DocData;\n type: string;\n widgetOpts: FormatOptions;\n visibleColType: string;\n visibleColWidgetOpts: FormatOptions;\n docSettings: DocumentSettings;\n}\n\n/**\n * Returns a constructor\n * with a format function that can properly convert a value passed to it into the\n * right format for that column.\n *\n * Pass fieldRef (a row ID of _grist_Views_section_field) to use the settings of that view field\n * instead of the table column.\n */\nexport function createFullFormatterFromDocData(\n docData: DocData,\n colRef: number,\n fieldRef?: number,\n): BaseFormatter {\n const [type, widgetOpts, docSettings] = createParserOrFormatterArguments(docData, colRef, fieldRef);\n const { visibleColType, visibleColWidgetOpts } = widgetOpts as ReferenceParsingOptions;\n return createFullFormatterRaw({\n docData,\n type,\n widgetOpts,\n visibleColType,\n visibleColWidgetOpts,\n docSettings,\n });\n}\n\nexport function createFullFormatterRaw(args: FullFormatterArgs) {\n const { type, widgetOpts, docSettings } = args;\n const visibleColFormatter = createVisibleColFormatterRaw(args);\n return createFormatter(type, { ...widgetOpts, visibleColFormatter }, docSettings);\n}\n\nexport function createVisibleColFormatterRaw(\n {\n docData,\n docSettings,\n type,\n visibleColType,\n visibleColWidgetOpts,\n widgetOpts,\n }: FullFormatterArgs,\n): BaseFormatter {\n let referencedTableId = gristTypes.getReferencedTableId(type);\n if (!referencedTableId) {\n return createFormatter(type, widgetOpts, docSettings);\n } else if (visibleColType) {\n return createFormatter(visibleColType, visibleColWidgetOpts, docSettings);\n } else {\n // This column displays the Row ID, e.g. Table1[2]\n // Make referencedTableId empty if the table is hidden\n const tablesData = docData.getMetaTable(\"_grist_Tables\");\n const tableRef = tablesData.findRow(\"tableId\", referencedTableId);\n if (isHiddenTable(tablesData, tableRef)) {\n referencedTableId = \"\";\n }\n return createFormatter(\"Id\", { tableId: referencedTableId }, docSettings);\n }\n}\n" }, { "path": "app/common/ValueGuesser.ts", "content": "import { CellValue } from \"app/common/DocActions\";\nimport { DocData } from \"app/common/DocData\";\nimport { DocumentSettings } from \"app/common/DocumentSettings\";\nimport { isObject } from \"app/common/gristTypes\";\nimport { countIf } from \"app/common/gutil\";\nimport { NumberFormatOptions } from \"app/common/NumberFormat\";\nimport NumberParse from \"app/common/NumberParse\";\nimport { dateTimeWidgetOptions, guessDateFormat } from \"app/common/parseDate\";\nimport { MetaRowRecord } from \"app/common/TableData\";\nimport { createFormatter } from \"app/common/ValueFormatter\";\nimport { createParserRaw, ValueParser } from \"app/common/ValueParser\";\n\nimport * as moment from \"moment-timezone\";\n\ninterface GuessedColInfo {\n type: string;\n widgetOptions?: object;\n}\n\nexport interface GuessResult {\n values?: CellValue[];\n colInfo: GuessedColInfo;\n}\n\ntype ColMetadata = Partial>;\n\nexport interface GuessColMetadata {\n values: CellValue[];\n colMetadata?: ColMetadata; // omitted if no changes are proposed.\n}\n\n/**\n * Class for guessing if an array of values should be interpreted as a specific column type.\n * T is the type of values that strings should be parsed to and is stored in the column.\n */\nabstract class ValueGuesser {\n /**\n * Guessed column type and maybe widget options.\n */\n public abstract colInfo(): GuessedColInfo;\n\n /**\n * Parse a single string to a typed value in such a way that formatting the value returns the original string.\n * If the string cannot be parsed, return the original string.\n */\n public abstract parse(value: string): T | string;\n\n /**\n * Attempt to parse at least 90% the string values losslessly according to the guessed colInfo.\n * Return null if this cannot be done.\n */\n public guess(values: (string | null)[], docSettings: DocumentSettings): GuessResult | null {\n const colInfo = this.colInfo();\n const { type, widgetOptions } = colInfo;\n const formatter = createFormatter(type, widgetOptions || {}, docSettings);\n const result: any[] = [];\n // max number of non-parsed strings to allow before giving up\n const maxUnparsed = countIf(values, v => Boolean(v)) * 0.1;\n let unparsed = 0;\n\n for (const value of values) {\n if (!value) {\n if (this.allowBlank()) {\n result.push(null);\n continue;\n } else {\n return null;\n }\n }\n\n const parsed = this.parse(value);\n // Give up if too many strings failed to parse or if the parsed value changes when converted back to text\n if ((typeof parsed === \"string\" && ++unparsed > maxUnparsed) ||\n !this.isEqualFormatted(formatter.formatAny(parsed), value)) {\n return null;\n }\n result.push(parsed);\n }\n return { values: result, colInfo };\n }\n\n /**\n * Whether this type of column can store nulls directly.\n */\n protected allowBlank(): boolean {\n return true;\n }\n\n protected isEqualFormatted(formatted1: string, formatted2: string): boolean {\n return formatted1 === formatted2;\n }\n}\n\nclass BoolGuesser extends ValueGuesser {\n public colInfo(): GuessedColInfo {\n return { type: \"Bool\" };\n }\n\n public parse(value: string): boolean | string {\n if (value === \"true\") {\n return true;\n } else if (value === \"false\") {\n return false;\n } else {\n return value;\n }\n }\n\n /**\n * This is the only type that can't store nulls, it converts them to false.\n */\n protected allowBlank(): boolean {\n return false;\n }\n}\n\nclass NumericGuesser extends ValueGuesser {\n private _parser: ValueParser;\n constructor(docSettings: DocumentSettings, private _options: NumberFormatOptions) {\n super();\n this._parser = createParserRaw(\"Numeric\", _options, docSettings);\n }\n\n public colInfo(): GuessedColInfo {\n const result: GuessedColInfo = { type: \"Numeric\" };\n if (Object.keys(this._options).length) {\n result.widgetOptions = this._options;\n }\n return result;\n }\n\n public parse(value: string): number | string {\n return this._parser.cleanParse(value);\n }\n\n protected isEqualFormatted(formatted1: string, formatted2: string): boolean {\n // Consider format guessing successful if it returns the typed-in numeric value exactly or\n // differing only in whitespace.\n formatted1 = formatted1.replace(NumberParse.removeCharsRegex, \"\");\n formatted2 = formatted2.replace(NumberParse.removeCharsRegex, \"\");\n return formatted1 === formatted2;\n }\n}\n\nclass DateGuesser extends ValueGuesser {\n // _format should be a full moment format string\n // _tz should be the document's default timezone\n constructor(private _format: string, private _tz: string) {\n super();\n }\n\n public colInfo(): GuessedColInfo {\n const widgetOptions = dateTimeWidgetOptions(this._format, false);\n let type;\n if (widgetOptions.timeFormat) {\n type = \"DateTime:\" + this._tz;\n } else {\n type = \"Date\";\n this._tz = \"UTC\";\n }\n return { widgetOptions, type };\n }\n\n // Note that this parsing is much stricter than parseDate to prevent loss of information.\n // Dates which can be parsed by parseDate based on the guessed widget options may not be parsed here.\n public parse(value: string): number | string {\n const m = moment.tz(value, this._format, true, this._tz);\n return m.isValid() ? m.valueOf() / 1000 : value;\n }\n}\n\nexport function guessColInfoWithDocData(values: (string | null)[], docData: DocData) {\n return guessColInfo(values, docData.docSettings(), docData.docInfo().timezone);\n}\n\nexport function guessColInfo(\n values: (string | null)[], docSettings: DocumentSettings, timezone: string,\n): GuessResult {\n // Use short-circuiting of || to only do as much work as needed,\n // in particular not guessing date formats before trying other types.\n return (\n new BoolGuesser()\n .guess(values, docSettings) ||\n new NumericGuesser(\n docSettings,\n NumberParse.fromSettings(docSettings).guessOptions(values),\n )\n .guess(values, docSettings) ||\n new DateGuesser(guessDateFormat(values, timezone), timezone)\n .guess(values, docSettings) ||\n // Don't return the same values back if there's no conversion to be done,\n // as they have to be serialized and transferred over a pipe to Python.\n { colInfo: { type: \"Text\" } }\n );\n}\n\n/**\n * Guess column info for a new column, returning the metadata suitable for using with AddTable or\n * AddColumn user actions. In particular, widgetOptions, if any, are returned as a JSON string.\n * Will suggest turning the column to an empty one if all the values are empty (null or \"\").\n */\nexport function guessColInfoForImports(values: CellValue[], docData: DocData): GuessColMetadata {\n if (values.every(v => (v === null || v === \"\"))) {\n // Suggest empty column.\n return { values, colMetadata: { type: \"Any\", isFormula: true, formula: \"\" } };\n }\n if (values.some(isObject)) {\n // Suggest no changes.\n return { values };\n }\n const strValues = values.map(v => (v === null || typeof v === \"string\" ? v : String(v)));\n const guessed = guessColInfoWithDocData(strValues, docData);\n values = guessed.values || values;\n\n const opts = guessed.colInfo.widgetOptions;\n const colMetadata: ColMetadata = { ...guessed.colInfo, widgetOptions: opts && JSON.stringify(opts) };\n if (!colMetadata.widgetOptions) {\n delete colMetadata.widgetOptions; // Omit widgetOptions unless it is actually valid JSON.\n }\n return { values, colMetadata };\n}\n" }, { "path": "app/common/ValueParser.ts", "content": "import { csvDecodeRow } from \"app/common/csvFormat\";\nimport { BulkColValues, CellValue, ColValues, UserAction } from \"app/common/DocActions\";\nimport { DocData } from \"app/common/DocData\";\nimport { DocumentSettings } from \"app/common/DocumentSettings\";\nimport * as gristTypes from \"app/common/gristTypes\";\nimport { getReferencedTableId, isFullReferencingType } from \"app/common/gristTypes\";\nimport * as gutil from \"app/common/gutil\";\nimport { safeJsonParse } from \"app/common/gutil\";\nimport { NumberFormatOptions } from \"app/common/NumberFormat\";\nimport NumberParse from \"app/common/NumberParse\";\nimport { parseDateStrict, parseDateTime } from \"app/common/parseDate\";\nimport { MetaRowRecord, TableData } from \"app/common/TableData\";\nimport { DateFormatOptions, DateTimeFormatOptions, formatDecoded, FormatOptions } from \"app/common/ValueFormatter\";\nimport { encodeObject } from \"app/plugin/objtypes\";\n\nimport flatMap from \"lodash/flatMap\";\nimport mapValues from \"lodash/mapValues\";\n\nexport class ValueParser {\n constructor(public type: string, public widgetOpts: FormatOptions, public docSettings: DocumentSettings) {\n }\n\n public cleanParse(value: string): any {\n if (!value) {\n return value;\n }\n return this.parse(value) ?? value;\n }\n\n public parse(value: string): any {\n return value;\n }\n}\n\nclass IdentityParser extends ValueParser {\n}\n\nexport class NumericParser extends ValueParser {\n private _parse: NumberParse;\n\n constructor(type: string, options: NumberFormatOptions, docSettings: DocumentSettings) {\n super(type, options, docSettings);\n this._parse = NumberParse.fromSettings(docSettings, options);\n }\n\n public parse(value: string): number | null {\n return this._parse.parse(value)?.result ?? null;\n }\n}\n\nclass DateParser extends ValueParser {\n public parse(value: string): any {\n return parseDateStrict(value, (this.widgetOpts as DateFormatOptions).dateFormat!);\n }\n}\n\nclass DateTimeParser extends ValueParser {\n constructor(type: string, widgetOpts: DateTimeFormatOptions, docSettings: DocumentSettings) {\n super(type, widgetOpts, docSettings);\n const timezone = gutil.removePrefix(type, \"DateTime:\") || \"\";\n this.widgetOpts = { ...widgetOpts, timezone };\n }\n\n public parse(value: string): any {\n return parseDateTime(value, this.widgetOpts);\n }\n}\n\nclass ChoiceListParser extends ValueParser {\n public cleanParse(value: string): string[] | null {\n value = value.trim();\n const result = (\n this._parseJson(value) ||\n this._parseCsv(value)\n ).map(v => v.trim())\n .filter(v => v);\n if (!result.length) {\n return null;\n }\n return [\"L\", ...result];\n }\n\n private _parseJson(value: string): string[] | undefined {\n // Don't parse JSON non-arrays\n if (value.startsWith(\"[\")) {\n const arr: unknown[] | null = safeJsonParse(value, null);\n return arr\n // Remove nulls and empty strings\n ?.filter(v => v || v === 0)\n // Convert values to strings, formatting nested JSON objects/arrays as JSON\n .map(v => formatDecoded(v));\n }\n }\n\n private _parseCsv(value: string): string[] {\n // Split everything on newlines which are not allowed by the choice editor.\n return flatMap(value.split(/[\\n\\r]+/), (row) => {\n return csvDecodeRow(row)\n .map(v => v.trim());\n });\n }\n}\n\n/**\n * This is different from other widget options which are simple JSON\n * stored on the field. These have to be specially derived\n * for referencing columns. See createParser.\n */\nexport interface ReferenceParsingOptions {\n visibleColId: string;\n visibleColType: string;\n visibleColWidgetOpts: FormatOptions;\n\n // If this is provided and loaded, the ValueParser will look up values directly.\n // Otherwise an encoded lookup will be produced for the data engine to handle.\n tableData?: TableData;\n}\n\nexport class ReferenceParser extends ValueParser {\n public widgetOpts: ReferenceParsingOptions;\n public tableData = this.widgetOpts.tableData;\n public visibleColParser = createParserRaw(\n this.widgetOpts.visibleColType,\n this.widgetOpts.visibleColWidgetOpts,\n this.docSettings,\n );\n\n protected _visibleColId = this.widgetOpts.visibleColId;\n\n public parse(raw: string): any {\n const value = this.visibleColParser.cleanParse(raw);\n return this.lookup(value, raw);\n }\n\n public lookup(value: any, raw: string): any {\n if (value == null || value === \"\" || !raw) {\n return 0; // default value for a reference column\n }\n\n if (this._visibleColId === \"id\") {\n const n = Number(value);\n if (Number.isInteger(n)) {\n value = n;\n // Don't return yet because we need to check that this row ID exists\n } else {\n return raw;\n }\n }\n\n if (!this.tableData?.isLoaded) {\n const options: { column: string, raw?: string } = { column: this._visibleColId };\n if (value !== raw) {\n options.raw = raw;\n }\n return [\"l\", value, options];\n }\n\n return this.tableData.findMatchingRowId({ [this._visibleColId]: value }) || raw;\n }\n}\n\nexport class ReferenceListParser extends ReferenceParser {\n public parse(raw: string): any {\n let values: any[] | null;\n try {\n values = JSON.parse(raw);\n } catch {\n values = null;\n }\n if (!Array.isArray(values)) {\n // csvDecodeRow should never raise an exception\n values = csvDecodeRow(raw);\n }\n values = values.map(v => typeof v === \"string\" ? this.visibleColParser.cleanParse(v) : encodeObject(v));\n\n if (!values.length || !raw) {\n return null; // null is the default value for a reference list column\n }\n\n if (this._visibleColId === \"id\") {\n const numbers = values.map(Number);\n if (numbers.every(Number.isInteger)) {\n values = numbers;\n // Don't return yet because we need to check that these row IDs exist\n } else {\n return raw;\n }\n }\n\n if (!this.tableData?.isLoaded) {\n const options: { column: string, raw?: string } = { column: this._visibleColId };\n if (!(values.length === 1 && values[0] === raw)) {\n options.raw = raw;\n }\n return [\"l\", values, options];\n }\n\n const rowIds: number[] = [];\n for (const value of values) {\n const rowId = this.tableData.findMatchingRowId({ [this._visibleColId]: value });\n if (rowId) {\n rowIds.push(rowId);\n } else {\n // There's no matching value in the visible column, i.e. this is not a valid reference.\n // We need to return a string which will become AltText.\n return raw;\n }\n }\n return [\"L\", ...rowIds];\n }\n}\n\nexport const valueParserClasses: { [type: string]: typeof ValueParser } = {\n Numeric: NumericParser,\n Int: NumericParser,\n Date: DateParser,\n DateTime: DateTimeParser,\n ChoiceList: ChoiceListParser,\n Ref: ReferenceParser,\n RefList: ReferenceListParser,\n Attachments: ReferenceListParser,\n};\n\n/**\n * Returns a ValueParser which can parse strings into values appropriate for\n * a specific widget field or table column.\n * widgetOpts is usually the field/column's widgetOptions JSON\n * but referencing columns need more than that, see ReferenceParsingOptions above.\n */\nexport function createParserRaw(\n type: string, widgetOpts: FormatOptions, docSettings: DocumentSettings,\n): ValueParser {\n const cls = valueParserClasses[gristTypes.extractTypeFromColType(type)] || IdentityParser;\n return new cls(type, widgetOpts, docSettings);\n}\n\n/**\n * Returns a ValueParser which can parse strings into values appropriate for\n * a specific widget field or table column.\n *\n * Pass fieldRef (a row ID of _grist_Views_section_field) to use the settings of that view field\n * instead of the table column.\n */\nexport function createParser(\n docData: DocData,\n colRef: number,\n fieldRef?: number,\n): ValueParser {\n return createParserRaw(...createParserOrFormatterArguments(docData, colRef, fieldRef));\n}\n\n/**\n * Returns arguments suitable for createParserRaw or createFormatter. Only for internal use.\n *\n * Pass fieldRef (a row ID of _grist_Views_section_field) to use the settings of that view field\n * instead of the table column.\n */\nexport function createParserOrFormatterArguments(\n docData: DocData,\n colRef: number,\n fieldRef?: number,\n): [string, object, DocumentSettings] {\n const columnsTable = docData.getMetaTable(\"_grist_Tables_column\");\n const fieldsTable = docData.getMetaTable(\"_grist_Views_section_field\");\n\n const col = columnsTable.getRecord(colRef)!;\n let fieldOrCol: MetaRowRecord<\"_grist_Tables_column\" | \"_grist_Views_section_field\"> = col;\n if (fieldRef) {\n const field = fieldsTable.getRecord(fieldRef);\n fieldOrCol = field?.widgetOptions ? field : col;\n }\n\n return createParserOrFormatterArgumentsRaw(docData, col.type, fieldOrCol.widgetOptions, fieldOrCol.visibleCol);\n}\n\nexport function createParserOrFormatterArgumentsRaw(\n docData: DocData,\n type: string,\n widgetOptions: string,\n visibleColRef: number,\n): [string, object, DocumentSettings] {\n const columnsTable = docData.getMetaTable(\"_grist_Tables_column\");\n const widgetOpts = safeJsonParse(widgetOptions, {});\n\n if (isFullReferencingType(type)) {\n const vcol = columnsTable.getRecord(visibleColRef);\n widgetOpts.visibleColId = vcol?.colId || \"id\";\n widgetOpts.visibleColType = vcol?.type;\n widgetOpts.visibleColWidgetOpts = safeJsonParse(vcol?.widgetOptions || \"\", {});\n widgetOpts.tableData = docData.getTable(getReferencedTableId(type)!);\n }\n\n return [type, widgetOpts, docData.docSettings()];\n}\n\n/**\n * Returns a copy of `colValues` with string values parsed according to the type and options of each column.\n * `bulk` should be `true` if `colValues` is of type `BulkColValues`.\n */\nfunction parseColValues(\n tableId: string, colValues: T, docData: DocData, bulk: boolean,\n): T {\n const columnsTable = docData.getMetaTable(\"_grist_Tables_column\");\n const tablesTable = docData.getMetaTable(\"_grist_Tables\");\n const tableRef = tablesTable.findRow(\"tableId\", tableId);\n if (!tableRef) {\n return colValues;\n }\n\n return mapValues(colValues, (values, colId) => {\n const colRef = columnsTable.findMatchingRowId({ colId, parentId: tableRef });\n if (!colRef) {\n // Column not found - let something else deal with that\n return values;\n }\n\n const parser = createParser(docData, colRef);\n\n // Optimisation: If there's no special parser for this column type, do nothing\n if (parser instanceof IdentityParser) {\n return values;\n }\n\n function parseIfString(val: any) {\n return typeof val === \"string\" ? parser.cleanParse(val) : val;\n }\n\n if (bulk) {\n if (!Array.isArray(values)) { // in case of bad input\n return values;\n }\n // `colValues` is of type `BulkColValues`\n return (values as CellValue[]).map(parseIfString);\n } else {\n // `colValues` is of type `ColValues`, `values` is just one value\n return parseIfString(values);\n }\n });\n}\n\nexport function parseUserAction(ua: UserAction, docData: DocData): UserAction {\n switch (ua[0]) {\n case \"AddRecord\":\n case \"UpdateRecord\":\n return _parseUserActionColValues(ua, docData, false);\n case \"BulkAddRecord\":\n case \"BulkUpdateRecord\":\n case \"ReplaceTableData\":\n return _parseUserActionColValues(ua, docData, true);\n case \"AddOrUpdateRecord\":\n // Parse `require` (2) and `col_values` (3). The action looks like:\n // ['AddOrUpdateRecord', table_id, require, col_values, options]\n // (`col_values` is called `fields` in the API)\n ua = _parseUserActionColValues(ua, docData, false, 2);\n ua = _parseUserActionColValues(ua, docData, false, 3);\n return ua;\n case \"BulkAddOrUpdateRecord\":\n ua = _parseUserActionColValues(ua, docData, true, 2);\n ua = _parseUserActionColValues(ua, docData, true, 3);\n return ua;\n default:\n return ua;\n }\n}\n\n// Returns a copy of the user action with one element parsed, by default the last one\nfunction _parseUserActionColValues(ua: UserAction, docData: DocData, parseBulk: boolean, index?: number,\n): UserAction {\n ua = ua.slice();\n const tableId = ua[1] as string;\n if (index === undefined) {\n index = ua.length - 1;\n }\n const colValues = ua[index] as ColValues | BulkColValues;\n ua[index] = parseColValues(tableId, colValues, docData, parseBulk);\n return ua;\n}\n" }, { "path": "app/common/WidgetOptions.ts", "content": "import { NumberFormatOptions } from \"app/common/NumberFormat\";\n\nexport interface WidgetOptions extends NumberFormatOptions {\n textColor?: \"string\";\n fillColor?: \"string\";\n alignment?: \"left\" | \"center\" | \"right\";\n dateFormat?: string;\n timeFormat?: string;\n widget?: \"HyperLink\";\n choices?: string[];\n}\n" }, { "path": "app/common/airtable/AirtableAPI.ts", "content": "import {\n AirtableBaseSchema,\n AirtableFieldSchema, AirtableListBasesResponse,\n AirtableTableSchema,\n} from \"app/common/airtable/AirtableAPITypes\";\nimport AirtableSchemaTypeSuite from \"app/common/airtable/AirtableAPITypes-ti\";\n\nimport Airtable, { Record, SelectOptions as QueryParams } from \"airtable\";\nimport { CheckerT, createCheckers } from \"ts-interface-checker\";\n\nexport interface AirtableAPIOptions {\n apiKey: string;\n endpointUrl?: string;\n}\n\n// TODO - Improve error handling. Airtable's API throws if an error response is returned,\n// but we don't want to show that directly to users.\n\n/**\n * Simplifies access to Airtable's API.\n * - Allows easy access to meta methods (e.g. schema retrieval, listing bases) that aren't exposed\n * by the \"airtable\" package.\n * - Applies type checking and assertions to the responses\n */\nexport class AirtableAPI {\n private readonly _airtable: Airtable;\n private _metaRequester: Airtable.Base;\n\n constructor(_options: AirtableAPIOptions) {\n this._airtable = new Airtable(_options);\n // Airtable's JS library doesn't support fetching schemas, but by passing an empty baseId\n // we can still force it to request the URL we want, and re-use the library's backoff logic\n // to help with Airtable's rate limiting.\n this._metaRequester = this._airtable.base(\"\");\n }\n\n public base(baseId: string) {\n return this._airtable.base(baseId);\n }\n\n public async listBases(): Promise {\n // Technically there's pagination here - but each request returns 1000 bases, so it feels\n // premature to implement.\n const response = await this._metaRequester.makeRequest({ path: `meta/bases` });\n const body = response.body as AirtableListBasesResponse;\n return body.bases;\n }\n\n public async getBaseSchema(baseId: string): Promise {\n const response = await this._metaRequester.makeRequest({\n path: `meta/bases/${baseId}/tables`,\n });\n const schema = response.body;\n if (!AirtableSchemaChecker.test(schema)) {\n throw new AirtableAPIError(\"unexpected response structure when fetching base schema\");\n }\n return schema;\n }\n}\n\nexport class AirtableAPIError extends Error {\n constructor(message: string) {\n super(`Airtable API error: ${message}`);\n }\n}\n\nexport interface ListAirtableRecordsResult {\n records: Airtable.Records,\n hasMoreRecords: boolean,\n fetchNextPage: FetchNextPageFunc\n}\n\ntype FetchNextPageFunc = () => Promise;\n\nconst fetchPageWhenNoMoreData: FetchNextPageFunc = () => Promise.resolve({\n records: [],\n hasMoreRecords: false,\n fetchNextPage: fetchPageWhenNoMoreData,\n});\n\n/**\n * Airtable's built-in record querying (base.table(\"MyTable\").select().eachPage()) is prone\n * to hanging indefinitely when an error is thrown from the callback, or if the callback fails to\n * call `nextPage()` correctly.\n *\n * This re-implements the listRecords functionality, while keeping the error handling,\n * rate-limiting and auth logic from the Airtable library.\n */\nexport function listRecords(\n base: Airtable.Base, tableName: string, params: QueryParams,\n): Promise {\n const table = base.table(tableName);\n\n const fetchNextPage = async (offset?: number): ReturnType => {\n const { body } = await base.makeRequest({\n method: \"GET\",\n path: `/${encodeURIComponent(tableName)}`,\n qs: {\n ...params,\n offset,\n },\n });\n\n const records = body.records.map((recordJson: string) => new Record(table, \"\", recordJson));\n const hasMoreRecords = body.offset !== undefined;\n\n return {\n records,\n hasMoreRecords,\n fetchNextPage: hasMoreRecords ? () => fetchNextPage(body.offset) : fetchPageWhenNoMoreData,\n };\n };\n\n return fetchNextPage();\n}\n\nconst checkers = createCheckers(AirtableSchemaTypeSuite);\nexport const AirtableSchemaChecker = checkers.AirtableBaseSchema as CheckerT;\nexport const AirtableSchemaTableChecker = checkers.AirtableSchemaTable as CheckerT;\nexport const AirtableSchemaFieldChecker = checkers.AirtableSchemaField as CheckerT;\n" }, { "path": "app/common/airtable/AirtableAPITypes-ti.ts", "content": "/**\n * This module was automatically generated by `ts-interface-builder`\n */\nimport * as t from \"ts-interface-checker\";\n// tslint:disable:object-literal-key-quotes\n\nexport const AirtableBaseId = t.name(\"string\");\n\nexport const AirtableTableId = t.name(\"string\");\n\nexport const AirtableFieldId = t.name(\"string\");\n\nexport const AirtableFieldName = t.name(\"string\");\n\nexport const AirtableBaseSchema = t.iface([], {\n \"tables\": t.array(\"AirtableTableSchema\"),\n});\n\nexport const AirtableTableSchema = t.iface([], {\n \"id\": \"AirtableTableId\",\n \"name\": \"string\",\n \"primaryFieldId\": \"string\",\n \"fields\": t.array(\"AirtableFieldSchema\"),\n});\n\nexport const AirtableFieldSchema = t.iface([], {\n \"id\": \"AirtableFieldId\",\n \"name\": \"AirtableFieldName\",\n \"type\": \"string\",\n \"options\": t.opt(t.iface([], {\n [t.indexKey]: \"any\",\n })),\n});\n\nexport const AirtableChoiceValue = t.iface([], {\n \"id\": \"string\",\n \"name\": \"string\",\n \"color\": \"string\",\n});\n\nexport const AirtableListBasesResponse = t.iface([], {\n \"bases\": t.array(t.iface([], {\n \"id\": \"string\",\n \"name\": \"string\",\n \"permissionLevel\": t.array(t.union(t.lit(\"none\"), t.lit(\"read\"), t.lit(\"comment\"), t.lit(\"edit\"), t.lit(\"create\"))),\n })),\n \"offset\": t.opt(\"string\"),\n});\n\nconst exportedTypeSuite: t.ITypeSuite = {\n AirtableBaseId,\n AirtableTableId,\n AirtableFieldId,\n AirtableFieldName,\n AirtableBaseSchema,\n AirtableTableSchema,\n AirtableFieldSchema,\n AirtableChoiceValue,\n AirtableListBasesResponse,\n};\nexport default exportedTypeSuite;\n" }, { "path": "app/common/airtable/AirtableAPITypes.ts", "content": "// Aliases for the various Airtable IDs makes various Map type definitions clearer.\nexport type AirtableBaseId = string;\nexport type AirtableTableId = string;\nexport type AirtableFieldId = string;\nexport type AirtableFieldName = string;\n\n// Airtable schema response. Limit this to only needed fields to minimise chance of breakage.\nexport interface AirtableBaseSchema {\n tables: AirtableTableSchema[];\n}\n\nexport interface AirtableTableSchema {\n id: AirtableTableId;\n name: string;\n primaryFieldId: string;\n fields: AirtableFieldSchema[];\n}\n\nexport interface AirtableFieldSchema {\n id: AirtableFieldId;\n name: AirtableFieldName;\n type: string;\n options?: { [key: string]: any };\n}\n\nexport interface AirtableChoiceValue {\n id: string;\n name: string;\n color: string;\n}\n\nexport interface AirtableListBasesResponse {\n bases: {\n id: string,\n name: string,\n permissionLevel: (\"none\" | \"read\" | \"comment\" | \"edit\" | \"create\")[],\n }[],\n offset?: string,\n}\n" }, { "path": "app/common/airtable/AirtableAttachmentTracker.ts", "content": "import { AirtableFieldSchema } from \"app/common/airtable/AirtableAPITypes\";\nimport { AirtableFieldMappingInfo, GristTableId } from \"app/common/airtable/AirtableCrosswalk\";\nimport { createEmptyBulkColValues } from \"app/common/airtable/AirtableReferenceTracker\";\nimport { TableColValues } from \"app/common/DocActions\";\nimport { getMaxUploadSizeAttachmentMB } from \"app/common/gristUrls\";\nimport { arrayRepeat, byteString } from \"app/common/gutil\";\nimport { GristObjCode } from \"app/plugin/GristData\";\n\nimport pick from \"lodash/pick\";\nimport pLimit from \"p-limit\";\n\nexport type AttachmentsByColumnId = Record;\n\ninterface Attachment {\n filename: string;\n size: number;\n url: string;\n}\n\ninterface AttachmentsForRecord {\n gristRecordId: number;\n attachmentsByColumnId: AttachmentsByColumnId;\n}\n\nexport class AttachmentTracker {\n private _tableAttachmentTrackers = new Map();\n\n public addTable(gristTableId: string, columnIdsToUpdate: string[]) {\n const tableTracker = new TableAttachmentTracker(gristTableId, columnIdsToUpdate);\n this._tableAttachmentTrackers.set(gristTableId, tableTracker);\n return tableTracker;\n }\n\n public getTables(): TableAttachmentTracker[] {\n return Array.from(this._tableAttachmentTrackers.values());\n }\n\n public getRemainingAttachmentsCount() {\n let count = 0;\n for (const table of this.getTables()) {\n count += table.getRemainingAttachmentsCount();\n }\n return count;\n }\n}\n\nexport class TableAttachmentTracker {\n private _attachmentsForRecords: AttachmentsForRecord[] = [];\n\n public constructor(private _tableId: string, private _columnIds: string[]) {\n }\n\n public addRecord(attachmentsForRecord: AttachmentsForRecord) {\n this._attachmentsForRecords.push(attachmentsForRecord);\n }\n\n public async importAttachments(\n uploadAttachment: (value: string | Blob, filename?: string) => Promise,\n updateRows: (tableId: GristTableId, rows: TableColValues) => Promise,\n options: {\n maxConcurrentUploads?: number;\n updateRowsBatchSize?: number;\n onBatchComplete?(): void;\n } = {},\n ) {\n const { maxConcurrentUploads = 5, updateRowsBatchSize = 25, onBatchComplete } = options;\n\n while (this._attachmentsForRecords.length > 0) {\n const attachmentsForRecords = this._attachmentsForRecords.splice(0, updateRowsBatchSize);\n const limit = pLimit(maxConcurrentUploads);\n const tableColValues: TableColValues = { id: [], ...createEmptyBulkColValues(this._columnIds) };\n const uploads: Promise[] = [];\n\n for (let rowIdx = 0; rowIdx < attachmentsForRecords.length; rowIdx++) {\n const { gristRecordId, attachmentsByColumnId } = attachmentsForRecords[rowIdx];\n\n tableColValues.id.push(gristRecordId);\n\n for (const colId of this._columnIds) {\n const attachments = attachmentsByColumnId[colId] ?? [];\n const cellValue: [GristObjCode.List, ...(number | undefined)[]] = [\n GristObjCode.List, ...arrayRepeat(attachments.length, undefined)];\n tableColValues[colId][rowIdx] = cellValue;\n\n attachments.forEach((attachment, index) => {\n uploads.push(\n limit(() => this._uploadAttachment(attachment, uploadAttachment)).then((id) => {\n cellValue[index + 1] = id;\n }),\n );\n });\n }\n }\n\n // TODO: Use a pipeline instead of batching for uploads. Batches are only as fast as the\n // slowest upload, and a particularly large attachment could hold up starting a new batch.\n // Also consider switching to allSettled and reporting any warnings/errors to the client.\n // Note that all errors are currently handled by _uploadAttachment, so this call shouldn't\n // throw.\n await Promise.all(uploads);\n\n for (const colId of this._columnIds) {\n for (let rowIdx = 0; rowIdx < attachmentsForRecords.length; rowIdx++) {\n const cellValue = tableColValues[colId][rowIdx] as [GristObjCode.List, ...(number | undefined)[]];\n const attachmentIds = cellValue.slice(1) as (number | undefined)[];\n tableColValues[colId][rowIdx] = [GristObjCode.List, ...attachmentIds.filter(id => id !== undefined)];\n }\n }\n\n await updateRows(this._tableId, tableColValues);\n\n onBatchComplete?.();\n }\n }\n\n public getRemainingAttachmentsCount() {\n let count = 0;\n for (const record of this._attachmentsForRecords) {\n for (const attachments of Object.values(record.attachmentsByColumnId)) {\n if (attachments) {\n count += attachments.length;\n }\n }\n }\n return count;\n }\n\n private async _uploadAttachment(\n { filename, size, url }: Attachment,\n uploadAttachment: (value: string | Blob, filename?: string) => Promise,\n ): Promise {\n try {\n const maxSize = getMaxUploadSizeAttachmentMB() * 1024 * 1024;\n if (size > maxSize) {\n throw new Error(`Attachments must not exceed ${byteString(maxSize)}`);\n }\n\n const response = await fetch(url);\n if (!response.ok) {\n throw new Error(`HTTP ${response.status}: ${response.statusText}`);\n }\n\n const blob = await response.blob();\n return await uploadAttachment(blob, filename);\n } catch (error) {\n console.error(`Failed to upload attachment \"${filename}\" (URL: ${url}):`, error);\n return undefined;\n }\n }\n}\n\nexport function isAttachmentField({ type }: AirtableFieldSchema) {\n return type === \"multipleAttachments\";\n}\n\nexport function extractAttachmentsFromRecordField(\n fieldValue: any,\n fieldMapping: AirtableFieldMappingInfo,\n): Attachment[] | undefined {\n if (fieldMapping.airtableField.type !== \"multipleAttachments\") { return undefined; }\n\n const attachments = Array.isArray(fieldValue) ? fieldValue : undefined;\n if (!attachments) { return undefined; }\n\n return attachments.map(a => pick(a, \"filename\", \"size\", \"url\"));\n}\n" }, { "path": "app/common/airtable/AirtableCrosswalk.ts", "content": "import {\n AirtableBaseSchema,\n AirtableFieldName, AirtableFieldSchema,\n AirtableTableId,\n AirtableTableSchema,\n} from \"app/common/airtable/AirtableAPITypes\";\nimport { AirtableIdColumnLabel } from \"app/common/airtable/AirtableSchemaImporter\";\nimport {\n ExistingColumnSchema,\n ExistingDocSchema,\n ExistingTableSchema,\n} from \"app/common/DocSchemaImportTypes\";\n\nexport type GristTableId = string;\n\nexport interface AirtableBaseSchemaCrosswalk {\n tables: Map\n}\n\nexport interface AirtableTableCrosswalk {\n airtableTable: AirtableTableSchema;\n gristTable: ExistingTableSchema;\n fields: Map\n // Special case - ID isn't a field in Airtable, but it's useful to have a mapping if it exists.\n airtableIdColumn?: ExistingColumnSchema;\n}\n\nexport interface AirtableFieldMappingInfo {\n airtableField: AirtableFieldSchema;\n gristColumn: ExistingColumnSchema;\n}\n\n/**\n * Creates a mapping from fields in an Airtable schema to fields in a Grist schema.\n * @param {AirtableBaseSchema} airtableSchema\n * @param {ExistingDocSchema} gristSchema\n * @param {Map} tableMap\n * @returns {{schemaCrosswalk: AirtableBaseSchemaCrosswalk, warnings: DocSchemaImportWarning[]}}\n */\nexport function createAirtableBaseToGristDocCrosswalk(\n airtableSchema: AirtableBaseSchema, gristSchema: ExistingDocSchema, tableMap: Map,\n): { schemaCrosswalk: AirtableBaseSchemaCrosswalk, warnings: AirtableCrosswalkWarning[] } {\n const schemaCrosswalk: AirtableBaseSchemaCrosswalk = {\n tables: new Map(),\n };\n const warnings: AirtableCrosswalkWarning[] = [];\n\n for (const [airtableTableId, gristTableId] of tableMap.entries()) {\n const airtableTableSchema = airtableSchema.tables.find(table => table.id === airtableTableId);\n const gristTableSchema = gristSchema.tables.find(table => table.id === gristTableId);\n\n if (!airtableTableSchema) {\n // Implementation error - this shouldn't be possible if the parameters are passed correctly.\n throw new Error(`No airtable table found with id '${airtableTableId}' when building crosswalk`);\n }\n\n if (!gristTableSchema) {\n warnings.push(new MissingGristTableWarning(gristTableId));\n continue;\n }\n\n const { crosswalk: tableCrosswalk, warnings: tableWarnings } =\n createAirtableTableToGristTableCrosswalk(airtableTableSchema, gristTableSchema);\n\n warnings.push(...tableWarnings);\n\n schemaCrosswalk.tables.set(airtableTableId, tableCrosswalk);\n }\n\n return {\n schemaCrosswalk,\n warnings,\n };\n}\n\nfunction createAirtableTableToGristTableCrosswalk(\n airtableTableSchema: AirtableTableSchema, gristTableSchema: ExistingTableSchema,\n) {\n const warnings: AirtableCrosswalkWarning[] = [];\n const crosswalk: AirtableTableCrosswalk = {\n airtableTable: airtableTableSchema,\n gristTable: gristTableSchema,\n fields: new Map(),\n airtableIdColumn: gristTableSchema.columns.find(column => column.label === AirtableIdColumnLabel),\n };\n\n for (const field of airtableTableSchema.fields) {\n // Match columns on label. It's the only reliable value we can automatically match on and the simplest to implement.\n const matchingColumn = findGristColumnForField(field, gristTableSchema);\n if (!matchingColumn) {\n warnings.push(new NoDestinationColumnWarning(gristTableSchema.id, field));\n continue;\n }\n // Airtable record queries list fields by name (not id), which is guaranteed to be unique.\n crosswalk.fields.set(field.name, {\n airtableField: field,\n gristColumn: matchingColumn,\n });\n }\n\n return { crosswalk, warnings };\n}\n\nfunction findGristColumnForField(field: AirtableFieldSchema, gristSchema: ExistingTableSchema) {\n return gristSchema.columns.find(column => column.label === field.name);\n}\n\nexport interface AirtableCrosswalkWarning {\n message: string;\n}\n\nclass MissingGristTableWarning implements AirtableCrosswalkWarning {\n public readonly message;\n\n constructor(public readonly tableId: AirtableTableId) {\n this.message = `No Grist table found with id '${tableId}'`;\n }\n}\n\nclass NoDestinationColumnWarning implements AirtableCrosswalkWarning {\n public readonly message: string;\n\n constructor(public readonly gristTableId: string, public readonly field: AirtableFieldSchema) {\n this.message = `No destination column in the Grist table '${gristTableId}' could be found for field '${field.name}'. A column with a matching label is required.`;\n }\n}\n" }, { "path": "app/common/airtable/AirtableDataImporter.ts", "content": "import { AirtableFieldSchema } from \"app/common/airtable/AirtableAPITypes\";\nimport {\n AttachmentsByColumnId,\n AttachmentTracker,\n extractAttachmentsFromRecordField,\n isAttachmentField,\n TableAttachmentTracker,\n} from \"app/common/airtable/AirtableAttachmentTracker\";\nimport { AirtableDataImportParams } from \"app/common/airtable/AirtableDataImporterTypes\";\nimport {\n createEmptyBulkColValues,\n extractRefFromRecordField,\n isRefField,\n ReferenceTracker,\n RefValuesByColumnId,\n TableReferenceTracker,\n} from \"app/common/airtable/AirtableReferenceTracker\";\nimport { BulkColValues, CellValue, GristObjCode } from \"app/plugin/GristData\";\n\nexport async function importDataFromAirtableBase(\n { listRecords, addRows, updateRows, uploadAttachment, schemaCrosswalk, onProgress }: AirtableDataImportParams,\n) {\n const referenceTracker = new ReferenceTracker();\n const attachmentTracker = new AttachmentTracker();\n\n const addRowsPromises: Promise[] = [];\n\n // TODO: Strings passed to onProgress calls in common code aren't translatable.\n onProgress?.({ percent: 0, status: \"Importing records from Airtable...\" });\n\n for (const [tableId, tableCrosswalk] of schemaCrosswalk.tables.entries()) {\n // Filter out any formula columns early - Grist will error on any write to formula columns.\n const fieldMappings = Array.from(tableCrosswalk.fields.values()).filter(mapping => !mapping.gristColumn.isFormula);\n const gristColumnIds = fieldMappings.map(mapping => mapping.gristColumn.id);\n\n // Airtable ID needs to be handled separately to fields, as it's not stored as a field in Airtable\n if (tableCrosswalk.airtableIdColumn) {\n gristColumnIds.push(tableCrosswalk.airtableIdColumn.id);\n }\n\n const referenceColumnIds = Array.from(tableCrosswalk.fields.values())\n .filter(mapping => isRefField(mapping.airtableField))\n .map(mapping => mapping.gristColumn.id);\n\n let tableReferenceTracker: TableReferenceTracker | undefined;\n if (referenceColumnIds.length > 0) {\n tableReferenceTracker = referenceTracker.addTable(tableCrosswalk.gristTable.id, referenceColumnIds);\n }\n\n const attachmentColumnIds = Array.from(tableCrosswalk.fields.values())\n .filter(mapping => isAttachmentField(mapping.airtableField))\n .map(mapping => mapping.gristColumn.id);\n\n let tableAttachmentTracker: TableAttachmentTracker | undefined;\n if (attachmentColumnIds.length > 0) {\n tableAttachmentTracker = attachmentTracker.addTable(tableCrosswalk.gristTable.id, attachmentColumnIds);\n }\n\n let listRecordsResult = await listRecords(tableId);\n\n while (listRecordsResult.records.length > 0) {\n const { records } = listRecordsResult;\n\n const colValues: BulkColValues = createEmptyBulkColValues(gristColumnIds);\n const airtableRecordIds: string[] = [];\n const refsByColumnIdForRecords: RefValuesByColumnId[] = [];\n const attachmentsByColumnIdForRecords: AttachmentsByColumnId[] = [];\n\n for (const record of records) {\n const refsByColumnId: RefValuesByColumnId = {};\n const attachmentsByColumnId: AttachmentsByColumnId = {};\n\n airtableRecordIds.push(record.id);\n for (const fieldMapping of fieldMappings) {\n const { airtableField, gristColumn } = fieldMapping;\n const rawFieldValue = record.fields[airtableField.name];\n\n if (isRefField(airtableField)) {\n refsByColumnId[gristColumn.id] = extractRefFromRecordField(rawFieldValue, fieldMapping);\n }\n\n if (isAttachmentField(airtableField)) {\n attachmentsByColumnId[gristColumn.id] = extractAttachmentsFromRecordField(rawFieldValue, fieldMapping);\n }\n\n if (isRefField(airtableField) || isAttachmentField(airtableField)) {\n // Column should remain blank until it's filled in by a later step.\n colValues[gristColumn.id].push(null);\n continue;\n }\n\n const converter =\n AirtableFieldValueConverters[fieldMapping.airtableField.type] ?? AirtableFieldValueConverters.identity;\n\n const value = converter(fieldMapping.airtableField, record.fields[fieldMapping.airtableField.name]);\n\n // Always push, even if the value is undefined, so that row values are always at the right index.\n colValues[fieldMapping.gristColumn.id].push(value ?? null);\n }\n\n if (tableCrosswalk.airtableIdColumn) {\n colValues[tableCrosswalk.airtableIdColumn.id].push(record.id);\n }\n\n refsByColumnIdForRecords.push(refsByColumnId);\n attachmentsByColumnIdForRecords.push(attachmentsByColumnId);\n }\n\n const addRowsPromise = addRows(tableCrosswalk.gristTable.id, colValues)\n .then((gristRowIds) => {\n airtableRecordIds.forEach((airtableRecordId, index) => {\n // Only add entries to the reference and attachment trackers once we know they're added to the table.\n referenceTracker.addRecordIdMapping(airtableRecordId, gristRowIds[index]);\n tableReferenceTracker?.addUnresolvedRecord({\n gristRecordId: gristRowIds[index],\n refsByColumnId: refsByColumnIdForRecords[index],\n });\n tableAttachmentTracker?.addRecord({\n gristRecordId: gristRowIds[index],\n attachmentsByColumnId: attachmentsByColumnIdForRecords[index],\n });\n });\n });\n\n addRowsPromises.push(addRowsPromise);\n\n listRecordsResult = await listRecordsResult.fetchNextPage();\n }\n }\n\n // Future improvement - report all errors here using Promise.allSettled, or continue even if\n // a few sets of rows throw errors\n await Promise.all(addRowsPromises);\n\n for (const tableReferenceTracker of referenceTracker.getTables()) {\n await tableReferenceTracker.bulkUpdateRowsWithUnresolvedReferences(updateRows);\n }\n\n const totalAttachmentsCount = attachmentTracker.getRemainingAttachmentsCount();\n for (const tableAttachmentTracker of attachmentTracker.getTables()) {\n await tableAttachmentTracker.importAttachments(\n uploadAttachment,\n updateRows,\n {\n onBatchComplete: () => {\n const remainingAttachmentsCount = attachmentTracker.getRemainingAttachmentsCount();\n const uploadedAttachmentsCount = totalAttachmentsCount - remainingAttachmentsCount;\n const attachmentsPercent = (uploadedAttachmentsCount / totalAttachmentsCount) * 100;\n onProgress?.({\n percent: 50 + (attachmentsPercent * 0.50),\n status: `Importing attachments from Airtable... (${remainingAttachmentsCount} remaining)`,\n });\n },\n },\n );\n }\n\n onProgress?.({ percent: 100 });\n}\n\ntype AirtableFieldValueConverter = (fieldSchema: AirtableFieldSchema, value: any) => CellValue | undefined;\nconst AirtableFieldValueConverters: Record = {\n identity(fieldSchema, value) {\n return value;\n },\n aiText(fieldSchema, aiTextState) {\n return aiTextState?.value;\n },\n createdBy(fieldSchema, collaborator) {\n return formatCollaborator(collaborator);\n },\n count(fieldSchema, collaborator) {\n throw new Error(\"Count is a formula column, and should not have data conversion run\");\n },\n formula(fieldSchema, collaborator) {\n throw new Error(\"Formula is a formula column, and should not have data conversion run\");\n },\n lastModifiedBy(fieldSchema, collaborator) {\n return formatCollaborator(collaborator);\n },\n lookup(fieldSchema, value) {\n // Lookup fields fetch values from other columns. This should be a formula in Grist, no value needed.\n throw new Error(\"Lookup is a formula column, and should not have data conversion run\");\n },\n multipleCollaborators(fieldSchema, collaborators) {\n const formattedCollaborators = collaborators?.map(formatCollaborator);\n if (!formattedCollaborators) { return null; }\n return formattedCollaborators.join(\", \");\n },\n singleCollaborator(fieldSchema, collaborator) {\n return formatCollaborator(collaborator);\n },\n multipleSelects(fieldSchema, choices?: string[]) {\n if (!choices) { return null; }\n return [GristObjCode.List, ...choices];\n },\n rollup(fieldSchema, collaborator) {\n throw new Error(\"Rollup is a formula column, and should not have data conversion run\");\n },\n};\n\nconst formatCollaborator = (collaborator: any) => collaborator?.name;\n" }, { "path": "app/common/airtable/AirtableDataImporterTypes.ts", "content": "import { ListAirtableRecordsResult } from \"app/common/airtable/AirtableAPI\";\nimport { AirtableTableId } from \"app/common/airtable/AirtableAPITypes\";\nimport { AirtableBaseSchemaCrosswalk, GristTableId } from \"app/common/airtable/AirtableCrosswalk\";\nimport { BulkColValues, TableColValues } from \"app/common/DocActions\";\n\n/**\n * Parameters for importing data from Airtable into Grist.\n */\nexport interface AirtableDataImportParams {\n // Airtable data API operations. Used to export data from Airtable.\n listRecords: ListRecordsFunc,\n\n // Grist data API operations. Used to import data into Grist.\n addRows: AddRowsFunc,\n updateRows: UpdateRowsFunc,\n uploadAttachment: UploadAttachmentFunc,\n\n // Mapping of Airtable tables to Grist tables.\n schemaCrosswalk: AirtableBaseSchemaCrosswalk,\n\n onProgress?(progress: AirtableImportProgress): void,\n}\n\n/**\n * The progress of an Airtable import.\n *\n * Used by the UI to show a progress bar with an optional status message communicating\n * the current operation in progress (e.g. importing attachments).\n */\nexport interface AirtableImportProgress {\n percent: number;\n status?: string;\n}\n\n/**\n * Function that fetches records from an Airtable table.\n */\nexport type ListRecordsFunc = (tableId: AirtableTableId) => Promise;\n\n/**\n * Function that adds rows to a Grist table.\n */\ntype AddRowsFunc = (tableId: GristTableId, rows: BulkColValues) => Promise;\n\n/**\n * Function that updates the column value(s) of a set of rows in a Grist table.\n */\nexport type UpdateRowsFunc = (tableId: GristTableId, rows: TableColValues) => Promise;\n\n/**\n * Function that uploads an attachment blob to Grist and returns the attachment ID.\n */\nexport type UploadAttachmentFunc = (value: string | Blob, filename?: string) => Promise;\n" }, { "path": "app/common/airtable/AirtableReferenceTracker.ts", "content": "import { AirtableFieldSchema } from \"app/common/airtable/AirtableAPITypes\";\nimport { AirtableFieldMappingInfo } from \"app/common/airtable/AirtableCrosswalk\";\nimport { UpdateRowsFunc } from \"app/common/airtable/AirtableDataImporterTypes\";\nimport { TableColValues } from \"app/common/DocActions\";\nimport { isNonNullish } from \"app/common/gutil\";\nimport { BulkColValues, GristObjCode } from \"app/plugin/GristData\";\n\nexport type RefValuesByColumnId = Record;\n\ninterface UnresolvedRefsForRecord {\n gristRecordId: number;\n refsByColumnId: RefValuesByColumnId;\n}\n\nexport class ReferenceTracker {\n // Maps known airtable ids to their grist row ids to enable reference resolution.\n // Airtable row ids are guaranteed unique within a base.\n private _rowIdLookup = new Map();\n // Group references by table and row to achieve bulk-updates and atomic resolutions for rows.\n private _tableReferenceTrackers = new Map();\n\n public addRecordIdMapping(originalRecordId: string, gristRecordId: number) {\n this._rowIdLookup.set(originalRecordId, gristRecordId);\n }\n\n public resolve(originalRecordId: string): number | undefined {\n return this._rowIdLookup.get(originalRecordId);\n }\n\n public addTable(gristTableId: string, columnIdsToUpdate: string[]) {\n const tableTracker = new TableReferenceTracker(this, gristTableId, columnIdsToUpdate);\n this._tableReferenceTrackers.set(gristTableId, tableTracker);\n return tableTracker;\n }\n\n public getTables(): TableReferenceTracker[] {\n return Array.from(this._tableReferenceTrackers.values());\n }\n}\n\n// Store and resolve references per-table to enable bulk updates.\nexport class TableReferenceTracker {\n private _unresolvedRefsForRecords: UnresolvedRefsForRecord[] = [];\n\n // To perform bulk updates, all reference columns need updating at the same time.\n // Enforce this by explicitly listing the column ids to use during instantiation.\n public constructor(private _parent: ReferenceTracker, private _tableId: string, private _columnIds: string[]) {\n }\n\n public addUnresolvedRecord(unresolvedRefsForRecord: UnresolvedRefsForRecord) {\n this._unresolvedRefsForRecords.push(unresolvedRefsForRecord);\n }\n\n public async bulkUpdateRowsWithUnresolvedReferences(\n updateRows: UpdateRowsFunc,\n options?: { batchSize?: number },\n ) {\n const batchSize = options?.batchSize ?? 100;\n\n let pendingUpdate: TableColValues = { id: [], ...createEmptyBulkColValues(this._columnIds) };\n\n for (const unresolvedRefsForRecord of this._unresolvedRefsForRecords) {\n pendingUpdate.id.push(unresolvedRefsForRecord.gristRecordId);\n\n // Every row needs an entry in its respective column in the bulk update, so always loop through\n // the same columns for every row.\n for (const columnId of this._columnIds) {\n const references = unresolvedRefsForRecord.refsByColumnId[columnId];\n // TODO - Unresolvable references are currently just skipped silently. Find a way to display\n // them in the cell / UI.\n const resolvedReferences = references ?\n references.map(originalRecordId => this._parent.resolve(originalRecordId)).filter(isNonNullish) : [];\n pendingUpdate[columnId].push(\n [GristObjCode.List, ...resolvedReferences],\n );\n }\n\n if (pendingUpdate.id.length >= batchSize) {\n await updateRows(this._tableId, pendingUpdate);\n pendingUpdate = { id: [], ...createEmptyBulkColValues(this._columnIds) };\n }\n }\n\n if (pendingUpdate.id.length > 0) {\n await updateRows(this._tableId, pendingUpdate);\n }\n }\n}\n\nexport function isRefField(field: AirtableFieldSchema) {\n return field.type === \"multipleRecordLinks\";\n}\n\nexport function extractRefFromRecordField(\n fieldValue: any,\n fieldMapping: AirtableFieldMappingInfo,\n): string[] | undefined {\n if (fieldMapping.airtableField.type === \"multipleRecordLinks\") {\n return fieldValue;\n }\n return undefined;\n}\n\nexport function createEmptyBulkColValues(columnIds: string[]): BulkColValues {\n return Object.fromEntries(columnIds.map(id => [id, []]));\n}\n" }, { "path": "app/common/airtable/AirtableSchemaImporter.ts", "content": "import {\n AirtableBaseSchema, AirtableChoiceValue,\n AirtableFieldSchema,\n AirtableTableSchema,\n} from \"app/common/airtable/AirtableAPITypes\";\nimport {\n ColumnImportSchema,\n DocSchemaImportWarning,\n FormulaTemplate,\n ImportSchema,\n OriginalTableRef,\n} from \"app/common/DocSchemaImport\";\nimport { RecalcWhen } from \"app/common/gristTypes\";\n\n/**\n * Design note: this needs to be deterministic and based solely on the Airtable base schema,\n * it should not be based on the current state of the Grist doc or any other parameters passed to\n * the import.\n *\n * Other areas of the import code may transform the created schema\n * (e.g. skipping tables, resolving references).\n * If this schema changes based on the destination document state, a user-given parameter or anything\n * not directly derived from the Airtable schema, the remainder of the import code may not adapt the\n * schema properly for the target document.\n */\nexport function gristDocSchemaFromAirtableSchema(\n baseSchema: AirtableBaseSchema,\n): { schema: ImportSchema; warnings: DocSchemaImportWarning[] } {\n const warnings: DocSchemaImportWarning[] = [];\n\n const schema: ImportSchema = {\n tables: baseSchema.tables.map((baseTable) => {\n const { columns, warnings: columnWarnings } =\n convertAirtableTableFieldsToColumnSchemas({ base: baseSchema, table: baseTable });\n\n warnings.push(...columnWarnings);\n\n return {\n originalId: baseTable.id,\n desiredGristId: baseTable.name,\n columns: [createAirtableIdColumnSchema(), ...columns],\n };\n }),\n };\n\n return { schema, warnings };\n}\n\nfunction convertAirtableTableFieldsToColumnSchemas(\n params: { base: AirtableBaseSchema, table: AirtableTableSchema },\n) {\n const { table } = params;\n const warnings: DocSchemaImportWarning[] = [];\n const columns = table.fields\n .map((field) => {\n const result = convertAirtableFieldToColumnSchema({ field, ...params });\n\n if (result.warning) {\n warnings.push(result.warning);\n }\n\n return result.column;\n })\n .filter((column): column is ColumnImportSchema => column !== undefined);\n\n return { columns, warnings };\n}\n\nfunction convertAirtableFieldToColumnSchema(\n params: { base: AirtableBaseSchema, table: AirtableTableSchema, field: AirtableFieldSchema },\n): { column?: ColumnImportSchema, warning?: DocSchemaImportWarning } {\n const { field, table, base } = params;\n\n if (!AirtableFieldMappers[field.type]) {\n return {\n column: undefined,\n warning: new UnsupportedFieldTypeWarning(field.type, field.name, { originalTableId: table.id }),\n };\n }\n return AirtableFieldMappers[field.type]({\n field,\n table,\n getTableIdForField: (fieldId: string) => findTableIdForField(base, fieldId),\n });\n}\n\nfunction findTableIdForField(baseSchema: AirtableBaseSchema, fieldId: string) {\n const tableId = baseSchema.tables.find(table => table.fields.find(field => field.id === fieldId))?.id;\n // Generally shouldn't happen - the schema should always have sufficient info to resolve a valid field id.\n if (tableId === undefined) {\n throw new Error(`Unable to resolve table id for Airtable field ${fieldId}`);\n }\n return tableId;\n}\n\nexport const AirtableIdColumnLabel = \"Airtable Id\";\nfunction createAirtableIdColumnSchema(): ColumnImportSchema {\n return {\n originalId: \"airtableId\",\n desiredGristId: \"Airtable Id\",\n type: \"Text\",\n label: AirtableIdColumnLabel,\n untieColIdFromLabel: true,\n };\n}\n\ninterface AirtableFieldMapperParams {\n field: AirtableFieldSchema,\n table: AirtableTableSchema,\n getTableIdForField: (fieldId: string) => string,\n}\n\ninterface AirtableFieldMapperResult {\n column: ColumnImportSchema,\n warning?: DocSchemaImportWarning,\n}\n\ntype AirtableFieldMapper = (params: AirtableFieldMapperParams) => AirtableFieldMapperResult;\nconst AirtableFieldMappers: { [type: string]: AirtableFieldMapper } = {\n aiText({ field }) {\n return {\n column: {\n originalId: field.id,\n desiredGristId: field.name,\n label: field.name,\n type: \"Text\",\n },\n };\n },\n autoNumber({ field, table }) {\n return {\n column: {\n originalId: field.id,\n desiredGristId: field.name,\n label: field.name,\n type: \"Numeric\",\n formula: {\n formula: \"MAX(PEEK([R0].all.[R1]))+1\",\n replacements: [\n { originalTableId: table.id },\n { originalTableId: table.id, originalColId: field.id },\n ],\n },\n },\n warning: new AutoNumberLimitationWarning(field.name, { originalTableId: table.id }),\n };\n },\n checkbox({ field }) {\n return {\n column: {\n originalId: field.id,\n desiredGristId: field.name,\n label: field.name,\n type: \"Bool\",\n },\n };\n },\n count({ field, table }) {\n let formula: FormulaTemplate = { formula: \"\", replacements: [] };\n const fieldOptions = field.options;\n if (fieldOptions?.isValid && fieldOptions.recordLinkFieldId) {\n formula = {\n formula: \"len($[R0])\",\n replacements: [{ originalTableId: table.id, originalColId: fieldOptions.recordLinkFieldId }],\n };\n }\n\n return {\n column: {\n originalId: field.id,\n desiredGristId: field.name,\n label: field.name,\n type: \"Numeric\",\n isFormula: true,\n formula,\n },\n warning: new CountLimitationWarning(field.name, { originalTableId: table.id }),\n };\n },\n createdBy({ field }) {\n return {\n column: {\n originalId: field.id,\n desiredGristId: field.name,\n label: field.name,\n type: \"Text\",\n },\n };\n },\n createdTime({ field }) {\n return {\n column: {\n originalId: field.id,\n desiredGristId: field.name,\n label: field.name,\n type: \"DateTime\",\n formula: { formula: \"NOW()\" },\n recalcWhen: RecalcWhen.DEFAULT,\n },\n };\n },\n currency({ field }) {\n return {\n column: {\n originalId: field.id,\n desiredGristId: field.name,\n label: field.name,\n type: \"Numeric\",\n widgetOptions: {\n // Airtable only provides a currency symbol, which is pretty useless for setting this column up.\n // Instead of showing a wrong currency - omit currency formatting and just use precision.\n decimals: field.options?.precision ?? 2,\n maxDecimals: field.options?.precision ?? 2,\n },\n },\n };\n },\n date({ field }) {\n return {\n column: {\n originalId: field.id,\n desiredGristId: field.name,\n label: field.name,\n type: \"Date\",\n widgetOptions: {\n isCustomDateFormat: true,\n // Airtable and Grist seem to share identical format syntax, based on limited testing\n dateFormat: field.options?.dateFormat?.format ?? \"MM/DD/YYYY\",\n },\n },\n };\n },\n dateTime({ field }) {\n return {\n column: {\n originalId: field.id,\n desiredGristId: field.name,\n label: field.name,\n type: \"DateTime\",\n widgetOptions: {\n isCustomDateFormat: true,\n // Airtable and Grist seem to share identical format syntax, based on limited testing\n dateFormat: field.options?.dateFormat?.format ?? \"MM/DD/YYYY\",\n isCustomTimeFormat: true,\n // Airtable and Grist seem to share identical format syntax, based on limited testing\n timeFormat: field.options?.timeFormat?.format ?? \"h:mma\",\n },\n },\n };\n },\n duration({ field, table }) {\n return {\n column: {\n originalId: field.id,\n desiredGristId: field.name,\n label: field.name,\n type: \"Numeric\",\n },\n warning: new DurationFormatWarning(field.name, { originalTableId: table.id }),\n };\n },\n email({ field }) {\n return {\n column: {\n originalId: field.id,\n desiredGristId: field.name,\n label: field.name,\n type: \"Text\",\n },\n };\n },\n formula({ field }) {\n const formula = typeof field.options?.formula === \"string\" ? field.options?.formula : \"No formula set\";\n // Store the formula as a comment to prevent it showing errors.\n const formattedFormula = formula.split(\"\\n\").map(line => `#${line.trim()}`).join(\"\\n\");\n return {\n column: {\n originalId: field.id,\n desiredGristId: field.name,\n label: field.name,\n // The field schema from Airtable has more information on what this should be,\n // such as field type, options and referenced fields.\n // The logic to implement that however doesn't seem worth the time investment.\n type: \"Any\",\n formula: { formula: formattedFormula },\n isFormula: true,\n },\n };\n },\n lastModifiedBy({ field }) {\n return {\n column: {\n originalId: field.id,\n desiredGristId: field.name,\n label: field.name,\n type: \"Text\",\n formula: { formula: 'user and f\"{user.Name}\"' },\n recalcWhen: 2,\n },\n };\n },\n lastModifiedTime({ field }) {\n return {\n column: {\n originalId: field.id,\n desiredGristId: field.name,\n label: field.name,\n type: \"DateTime\",\n formula: { formula: \"NOW()\" },\n recalcWhen: 2,\n widgetOptions: {\n isCustomDateFormat: true,\n dateFormat: field.options?.result?.dateFormat?.format ?? \"MM/DD/YYYY\",\n isCustomTimeFormat: true,\n timeFormat: field.options?.result?.timeFormat?.format ?? \"h:mma\",\n },\n },\n };\n },\n multilineText({ field }) {\n return {\n column: {\n originalId: field.id,\n desiredGristId: field.name,\n label: field.name,\n type: \"Text\",\n },\n };\n },\n multipleAttachments({ field }) {\n return {\n column: {\n originalId: field.id,\n desiredGristId: field.name,\n label: field.name,\n type: \"Attachments\",\n },\n };\n },\n multipleCollaborators({ field }) {\n return {\n column: {\n originalId: field.id,\n desiredGristId: field.name,\n label: field.name,\n type: \"Text\",\n // Do we make a collaborators table and make this a reference instead?\n },\n };\n },\n multipleLookupValues({ field, table, getTableIdForField }) {\n let formula: FormulaTemplate = { formula: \"\" };\n const fieldOptions = field.options;\n if (fieldOptions?.recordLinkFieldId && fieldOptions.fieldIdInLinkedTable) {\n formula = {\n formula: \"$[R0].[R1]\",\n replacements: [\n { originalTableId: table.id, originalColId: fieldOptions.recordLinkFieldId },\n {\n originalTableId: getTableIdForField(fieldOptions.fieldIdInLinkedTable),\n originalColId: fieldOptions.fieldIdInLinkedTable,\n },\n ],\n };\n }\n return {\n column: {\n originalId: field.id,\n desiredGristId: field.name,\n label: field.name,\n type: \"Any\",\n isFormula: true,\n formula,\n },\n };\n },\n multipleRecordLinks({ field }) {\n return {\n column: {\n originalId: field.id,\n desiredGristId: field.name,\n label: field.name,\n type: field.options?.prefersSingleRecordLink ? \"Ref\" : \"RefList\",\n ref: {\n originalTableId: field.options?.linkedTableId,\n },\n },\n };\n },\n multipleSelects({ field }) {\n return {\n column: {\n originalId: field.id,\n desiredGristId: field.name,\n label: field.name,\n type: \"ChoiceList\",\n widgetOptions: {\n choices: field.options?.choices.map((choice: AirtableChoiceValue) => choice.name),\n // We could import the color by mapping choice.color (e.g. tealLight2) to a hex color\n choiceOptions: {},\n },\n },\n };\n },\n number({ field }) {\n return {\n column: {\n originalId: field.id,\n desiredGristId: field.name,\n label: field.name,\n type: \"Numeric\",\n widgetOptions: {\n decimals: field.options?.precision,\n },\n },\n };\n },\n percent({ field }) {\n return {\n column: {\n originalId: field.id,\n desiredGristId: field.name,\n label: field.name,\n type: \"Numeric\",\n widgetOptions: {\n decimals: field.options?.precision,\n numMode: \"percent\",\n },\n },\n };\n },\n phoneNumber({ field }) {\n return {\n column: {\n originalId: field.id,\n desiredGristId: field.name,\n label: field.name,\n type: \"Text\",\n },\n };\n },\n rating({ field }) {\n return {\n column: {\n originalId: field.id,\n desiredGristId: field.name,\n label: field.name,\n type: \"Int\",\n // Consider setting up some nice conditional formatting.\n },\n };\n },\n richText({ field }) {\n return {\n column: {\n originalId: field.id,\n desiredGristId: field.name,\n label: field.name,\n type: \"Text\",\n widgetOptions: {\n widget: \"Markdown\",\n },\n },\n };\n },\n rollup({ field, table, getTableIdForField }) {\n let formula: FormulaTemplate = { formula: \"\" };\n const fieldOptions = field.options;\n if (fieldOptions?.recordLinkFieldId && fieldOptions.fieldIdInLinkedTable) {\n formula = {\n formula: \"$[R0].[R1]\",\n replacements: [\n { originalTableId: table.id, originalColId: fieldOptions.recordLinkFieldId },\n {\n originalTableId: getTableIdForField(fieldOptions.fieldIdInLinkedTable),\n originalColId: fieldOptions.fieldIdInLinkedTable,\n },\n ],\n };\n }\n return {\n column: {\n originalId: field.id,\n desiredGristId: field.name,\n label: field.name,\n type: \"Any\",\n isFormula: true,\n formula,\n },\n warning: new RollupLimitationWarning(field.name, { originalTableId: table.id }),\n };\n },\n singleCollaborator({ field }) {\n return {\n column: {\n originalId: field.id,\n desiredGristId: field.name,\n label: field.name,\n type: \"Text\",\n },\n };\n },\n singleLineText({ field }) {\n return {\n column: {\n originalId: field.id,\n desiredGristId: field.name,\n label: field.name,\n type: \"Text\",\n // We could potentially limit this to only a single line, but it's a view section option\n // which isn't (at the time of writing) supported by any of the import tools (which only deal\n // with structure, e.g. tables and columns).\n },\n };\n },\n singleSelect({ field }) {\n return {\n column: {\n originalId: field.id,\n desiredGristId: field.name,\n label: field.name,\n type: \"Choice\",\n widgetOptions: {\n choices: field.options?.choices.map((choice: AirtableChoiceValue) => choice.name),\n // We could import the color by mapping choice.color (e.g. tealLight2) to a hex color\n choiceOptions: {},\n },\n },\n };\n },\n url({ field }) {\n return {\n column: {\n originalId: field.id,\n desiredGristId: field.name,\n label: field.name,\n type: \"Text\",\n widgetOptions: {\n widget: \"HyperLink\",\n },\n },\n };\n },\n};\n\nclass UnsupportedFieldTypeWarning implements DocSchemaImportWarning {\n public readonly message: string;\n\n constructor(fieldType: string, fieldName: string, public readonly ref: OriginalTableRef) {\n this.message = `Field \"${fieldName}\" has unsupported type \"${fieldType}\" and will be skipped`;\n }\n}\n\nclass AutoNumberLimitationWarning implements DocSchemaImportWarning {\n public readonly message: string;\n\n constructor(fieldName: string, public readonly ref: OriginalTableRef) {\n this.message = `AutoNumber field \"${fieldName}\" behaviour will not be identical to Airtable's. Values may be re-used if rows are edited or deleted.`;\n }\n}\n\nclass DurationFormatWarning implements DocSchemaImportWarning {\n public readonly message: string;\n\n constructor(fieldName: string, public readonly ref: OriginalTableRef) {\n this.message = `Duration field \"${fieldName}\" will be imported as a numeric duration in seconds. Duration formatting is not yet supported.`;\n }\n}\n\nclass RollupLimitationWarning implements DocSchemaImportWarning {\n public readonly message: string;\n\n constructor(fieldName: string, public readonly ref: OriginalTableRef) {\n this.message = `Rollup field \"${fieldName}\" may not match Airtable. Summary parameters and filter conditions are not supported.`;\n }\n}\n\nclass CountLimitationWarning implements DocSchemaImportWarning {\n public readonly message: string;\n\n constructor(fieldName: string, public readonly ref: OriginalTableRef) {\n this.message = `Count field \"${fieldName}\" may not match Airtable. Filter conditions are not supported.`;\n }\n}\n" }, { "path": "app/common/arrayToString.ts", "content": "/**\n * Functions to convert between an array of bytes and a string. The implementations are\n * different for Node and for the browser.\n */\n\ndeclare const TextDecoder: any, TextEncoder: any;\n\nexport let arrayToString: (data: Uint8Array) => string;\nexport let stringToArray: (data: string) => Uint8Array;\n\nif (typeof TextDecoder !== \"undefined\") {\n // Note that constructing a TextEncoder/Decoder takes time, so it's faster to reuse.\n const dec = new TextDecoder(\"utf8\");\n const enc = new TextEncoder(\"utf8\");\n arrayToString = function(uint8Array: Uint8Array): string {\n return dec.decode(uint8Array);\n };\n stringToArray = function(str: string): Uint8Array {\n return enc.encode(str);\n };\n} else {\n arrayToString = function(uint8Array: Uint8Array): string {\n return Buffer.from(uint8Array).toString(\"utf8\");\n };\n stringToArray = function(str: string): Uint8Array {\n return new Uint8Array(Buffer.from(str, \"utf8\"));\n };\n}\n" }, { "path": "app/common/asyncIterators.ts", "content": "/**\n * Just some basic utilities for async generators that should really be part of the language or lodash or something.\n */\n\nexport async function* asyncFilter(it: AsyncIterableIterator, pred: (x: T) => boolean): AsyncIterableIterator {\n for await (const x of it) {\n if (pred(x)) {\n yield x;\n }\n }\n}\n\nexport async function* asyncMap(it: AsyncIterableIterator, mapper: (x: T) => R): AsyncIterableIterator {\n for await (const x of it) {\n yield mapper(x);\n }\n}\n\nexport async function toArray(it: AsyncIterableIterator): Promise {\n const result = [];\n for await (const x of it) {\n result.push(x);\n }\n return result;\n}\n" }, { "path": "app/common/csvFormat.ts", "content": "/**\n * Simple utilities for escaping/quoting/parsing CSV data.\n *\n * This only supports the default Excel-like encoding, in which fields containing any separators\n * or quotes get quoted (using '\"'), and quotes get doubled.\n *\n * Quoting is also applied when values contain leading or trailing whitespace, and on parsing,\n * leading or trailing whitespace in unquoted values is trimmed, so that \",\" or \", \" may be used\n * as a separator.\n *\n * This is intended for copy-pasting multi-choice values, where plain comma-separated text is the\n * most user-friendly, and CSV encoding is used to ensure we can handle arbitrary values.\n */\n\n// Encode a row. If {prettier: true} is set, separate output with \", \". Leading whitespace gets\n// encoded in any case.\nexport function csvEncodeRow(values: string[], options: { prettier?: boolean } = {}): string {\n return values.map(csvEncodeCell).join(options.prettier ? \", \" : \",\");\n}\n\nexport function csvDecodeRow(text: string): string[] {\n // Clever regexp from https://github.com/micnews/csv-line\n const parts = text.split(/((?:(?:\"[^\"]*\")|[^,])*)/);\n const main = parts.filter((v, idx) => idx % 2).map(csvDecodeCell);\n // The \"delimiter\" (odd-numbered parts) is our content. If it's not at the start/end, it means\n // we have commas, and should include empty fields at those ends.\n if (parts[0]) { main.unshift(\"\"); }\n if (parts[parts.length - 1]) { main.push(\"\"); }\n return main;\n}\n\nexport function csvEncodeCell(value: string): string {\n return /[,\\r\\n\"]|^\\s|\\s$/.test(value) ? '\"' + value.replace(/\"/g, '\"\"') + '\"' : value;\n}\n\nexport function csvDecodeCell(value: string): string {\n return value.trim().replace(/^\"|\"$/g, \"\").replace(/\"\"/g, '\"');\n}\n" }, { "path": "app/common/declarations.d.ts", "content": "declare module \"app/common/MemBuffer\" {\n const MemBuffer: any;\n type MemBuffer = any;\n export = MemBuffer;\n}\n\ndeclare module \"locale-currency/map\" {\n const Map: Record;\n type Map = Record;\n export = Map;\n}\n\ndeclare namespace Intl {\n class DisplayNames {\n public static supportedLocalesOf(locales: string | string[]): string[];\n constructor(locales?: string, options?: object);\n public of(code: string): string;\n }\n\n class Locale {\n public region: string;\n public language: string;\n constructor(locale: string);\n }\n}\n\ndeclare module \"@gristlabs/moment-guess/dist/bundle.js\";\n" }, { "path": "app/common/delay.ts", "content": "/**\n * Returns a promise that resolves in the given number of milliseconds.\n * (A replica of bluebird.delay using native promises.)\n */\nexport function delay(msec: number): Promise {\n return new Promise(resolve => setTimeout(resolve, msec));\n}\n" }, { "path": "app/common/emails.ts", "content": "/**\n *\n * Utilities related to email normalization. Currently\n * trivial, but could potentially need special per-domain\n * rules in future.\n *\n * Email addresses are a bit slippery. Domain names are\n * case insensitive, but user names may or may not be,\n * depending on the mail server handling the domain.\n * Other special treatment of user names may also be in\n * place for particular domains (periods, plus sign, etc).\n *\n * We treat emails as case-insensitive for the purposes\n * of determining equality of emails, and indexing users\n * by email address.\n *\n */\n\n/**\n *\n * Convert the supplied email address to a normalized form\n * that we will use for indexing and equality tests.\n * Many possible email addresses could map to the same\n * normalized result; as far as we are concerned those\n * addresses are equivalent.\n *\n * The normalization we do is a simple lowercase. This\n * means we won't be able to treat both Jane@x.y and\n * jane@x.y as separate email addresses, even through\n * they may in fact be separate mailboxes on x.y.\n *\n * The normalized email is not something we should show\n * the user in the UI, but is rather for internal purposes.\n *\n * The original non-normalized email is called a\n * \"display email\" to distinguish it from a \"normalized\n * email\"\n *\n */\nexport function normalizeEmail(displayEmail: string): string {\n // We take the lower case, without use of locale.\n return displayEmail.toLowerCase();\n}\n" }, { "path": "app/common/getCurrentTime.ts", "content": "import moment from \"moment-timezone\";\n\n/**\n * Returns the current local time. Allows overriding via a \"currentTime\" URL parameter, for the sake\n * of tests.\n */\nexport default function getCurrentTime(): moment.Moment {\n const getDefault = () => moment();\n if (typeof window === \"undefined\" || !window) { return getDefault(); }\n const searchParams = new URLSearchParams(window.location.search);\n\n return searchParams.has(\"currentTime\") ? moment(searchParams.get(\"currentTime\") || undefined) : getDefault();\n}\n" }, { "path": "app/common/gristTypes.ts", "content": "import { CellValue, CellVersions } from \"app/common/DocActions\";\nimport { removePrefix } from \"app/common/gutil\";\nimport { GristObjCode, GristType } from \"app/plugin/GristData\";\n\nimport isString from \"lodash/isString\";\n\nexport type GristTypeInfo =\n { type: \"DateTime\", timezone: string } |\n { type: \"Ref\", tableId: string } |\n { type: \"RefList\", tableId: string } |\n { type: Exclude };\n\nexport const MANUALSORT = \"manualSort\";\n\n// Whether a column is internal and should be hidden.\nexport function isHiddenCol(colId: string): boolean {\n return colId.startsWith(\"gristHelper_\") || colId === MANUALSORT;\n}\n\n// This mapping includes both the default value, and its representation for SQLite.\nconst _defaultValues: { [key in GristType]: [CellValue, string] } = {\n Any: [null, \"NULL\"],\n Attachments: [null, \"NULL\"],\n Blob: [null, \"NULL\"],\n // Bool is only supported by SQLite as 0 and 1 values.\n Bool: [false, \"0\"],\n Choice: [\"\", \"''\"],\n ChoiceList: [null, \"NULL\"],\n Date: [null, \"NULL\"],\n DateTime: [null, \"NULL\"],\n Id: [0, \"0\"],\n Int: [0, \"0\"],\n // Note that \"1e999\" is a way to store Infinity into SQLite. This is verified by \"Defaults\"\n // tests in DocStorage.js. See also http://sqlite.1065341.n5.nabble.com/Infinity-td55327.html.\n ManualSortPos: [Number.POSITIVE_INFINITY, \"1e999\"],\n Numeric: [0, \"0\"],\n PositionNumber: [Number.POSITIVE_INFINITY, \"1e999\"],\n Ref: [0, \"0\"],\n RefList: [null, \"NULL\"],\n Text: [\"\", \"''\"],\n};\n\n/**\n * Given a grist column type (e.g Text, Numeric, ...) returns the default value for that type.\n * If options.sqlFormatted is true, returns the representation of the value for SQLite.\n */\nexport function getDefaultForType(colType: string, options: { sqlFormatted?: boolean } = {}) {\n const type = extractTypeFromColType(colType);\n return (_defaultValues[type as GristType] || _defaultValues.Any)[options.sqlFormatted ? 1 : 0];\n}\n\n/**\n * Convert a type like 'Numeric', 'DateTime:America/New_York', or 'Ref:Table1' to a GristTypeInfo\n * object.\n */\nexport function extractInfoFromColType(colType: string): GristTypeInfo {\n if (colType === \"Attachments\") {\n return { type: \"RefList\", tableId: \"_grist_Attachments\" };\n }\n const colon = colType.indexOf(\":\");\n const [type, arg] = (colon === -1) ? [colType] : [colType.slice(0, colon), colType.slice(colon + 1)];\n return (type === \"Ref\") ? { type, tableId: String(arg) } :\n (type === \"RefList\") ? { type, tableId: String(arg) } :\n (type === \"DateTime\") ? { type, timezone: String(arg) } :\n { type } as GristTypeInfo;\n}\n\n/**\n * Re-encodes a CellValue of a given Grist type as a value suitable to use in an Any column. E.g.\n * reencodeAsTypedCellValue(123, {type: \"Numeric\"}) -> 123\n * reencodeAsTypedCellValue(123, {type: \"Date\"}) -> [\"d\", 123]\n * reencodeAsTypedCellValue(123, {type: \"Ref\", tableId: \"Table1\"}) -> [\"R\", \"Table1\", 123]\n * reencodeAsTypedCellValue([\"L\", 123], {type: \"RefList\", tableId: \"Table1\"}) -> [\"r\", \"Table1\", [123]]\n * reencodeAsTypedCellValue([\"L\", 123], {type: \"Attachments\"}) -> [\"r\", \"_grist_Attachments\", [123]]\n */\nexport function reencodeAsTypedCellValue(value: CellValue, typeInfo: GristTypeInfo): CellValue {\n if (typeof value === \"number\") {\n switch (typeInfo.type) {\n case \"Date\": return [GristObjCode.Date, value];\n case \"DateTime\": return [GristObjCode.DateTime, value, typeInfo.timezone];\n case \"Ref\": return [GristObjCode.Reference, typeInfo.tableId, value];\n }\n } else if (isList(value) || value === null) {\n const items = value ? value.slice(1) : [];\n switch (typeInfo.type) {\n case \"ChoiceList\": return [GristObjCode.List, ...items];\n case \"RefList\": return [GristObjCode.ReferenceList, typeInfo.tableId, items];\n case \"Attachments\": return [GristObjCode.ReferenceList, \"_grist_Attachments\", items];\n }\n }\n return value;\n}\n\n/**\n * Returns whether a value (as received in a DocAction) represents a custom object.\n */\nexport function isObject(value: CellValue): value is [GristObjCode, any?] {\n return Array.isArray(value);\n}\n\n/**\n * Returns GristObjCode of the value if the value is an object, or null otherwise.\n * The return type includes any string, since we should not assume we can only get valid codes.\n */\nexport function getObjCode(value: CellValue): GristObjCode | string | null {\n return Array.isArray(value) ? value[0] : null;\n}\n\n/**\n * Returns whether a value (as received in a DocAction) represents a raised exception.\n */\nexport function isRaisedException(value: CellValue): boolean {\n return getObjCode(value) === GristObjCode.Exception;\n}\n\n/**\n * Returns whether a value (as received in a DocAction) represents a group of versions for\n * a comparison or conflict.\n */\nexport function isVersions(value: CellValue): value is [GristObjCode.Versions, CellVersions] {\n return getObjCode(value) === GristObjCode.Versions;\n}\n\nexport function isSkip(value: CellValue): value is [GristObjCode.Skip] {\n return getObjCode(value) === GristObjCode.Skip;\n}\n\nexport function isCensored(value: CellValue): value is [GristObjCode.Censored] {\n return getObjCode(value) === GristObjCode.Censored;\n}\n\n/**\n * Returns whether a value (as received in a DocAction) represents a list.\n */\nexport function isList(value: CellValue): value is [GristObjCode.List, ...CellValue[]] {\n return Array.isArray(value) && value[0] === GristObjCode.List;\n}\n\n/**\n * Returns whether a value (as received in a DocAction) represents a reference to a record.\n */\nexport function isReference(value: CellValue): value is [GristObjCode.Reference, string, number] {\n return Array.isArray(value) && value[0] === GristObjCode.Reference;\n}\n\n/**\n * Returns whether a value (as received in a DocAction) represents a reference list (RecordSet).\n */\nexport function isReferenceList(value: CellValue): value is [GristObjCode.ReferenceList, string, number[]] {\n return Array.isArray(value) && value[0] === GristObjCode.ReferenceList;\n}\n\n/**\n * Returns whether a value (as received in a DocAction) represents a reference or reference list.\n */\nexport function isReferencing(value: CellValue):\n value is [GristObjCode.ReferenceList | GristObjCode.Reference, string, number[] | number] {\n return Array.isArray(value) &&\n (value[0] === GristObjCode.ReferenceList || value[0] === GristObjCode.Reference);\n}\n\n/**\n * Returns whether a value (as received in a DocAction) represents a list or is null,\n * which is a valid value for list types in grist.\n */\nexport function isListOrNull(value: CellValue): boolean {\n return value === null || isList(value);\n}\n\n/**\n * Returns whether a value (as received in a DocAction) represents an empty list.\n */\nexport function isEmptyList(value: CellValue): boolean {\n return Array.isArray(value) && value.length === 1 && value[0] === GristObjCode.List;\n}\n\n/**\n * Returns whether a value (as received in a DocAction) represents an empty reference list.\n */\nexport function isEmptyReferenceList(value: CellValue): boolean {\n return Array.isArray(value) && value.length === 1 && value[0] === GristObjCode.ReferenceList;\n}\n\nfunction isNumber(v: CellValue) { return typeof v === \"number\" || typeof v === \"boolean\"; }\nfunction isNumberOrNull(v: CellValue) { return isNumber(v) || v === null; }\nfunction isBoolean(v: CellValue) { return typeof v === \"boolean\" || v === 1 || v === 0; }\nfunction isBooleanOrNull(v: CellValue) { return isBoolean(v) || v === null; }\n\n// These values are not regular cell values, even in a column of type Any.\nconst abnormalValueTypes: string[] = [GristObjCode.Exception, GristObjCode.Pending, GristObjCode.Skip,\n GristObjCode.Unmarshallable, GristObjCode.Versions];\n\nfunction isNormalValue(value: CellValue) {\n return !abnormalValueTypes.includes(getObjCode(value)!);\n}\n\n/**\n * Map of Grist type to an \"isRightType\" checker function, which determines if a given values type\n * matches the declared type of the column.\n */\nconst rightType: { [key in GristType]: (value: CellValue) => boolean } = {\n Any: isNormalValue,\n Attachments: isListOrNull,\n Text: isString,\n Blob: isString,\n Int: isNumberOrNull,\n Bool: isBooleanOrNull,\n Date: isNumberOrNull,\n DateTime: isNumberOrNull,\n Numeric: isNumberOrNull,\n Id: isNumber,\n PositionNumber: isNumber,\n ManualSortPos: isNumber,\n Ref: isNumber,\n RefList: isListOrNull,\n Choice: isString,\n ChoiceList: isListOrNull,\n};\n\nexport function isRightType(type: string): undefined | ((value: CellValue, options?: any) => boolean) {\n return rightType[type as GristType];\n}\n\nexport function extractTypeFromColType(type: string): string {\n if (!type) { return type; }\n const colon = type.indexOf(\":\");\n return (colon === -1 ? type : type.slice(0, colon));\n}\n\n/**\n * Enum for values of columns' recalcWhen property, corresponding to Python definitions in\n * schema.py.\n */\nexport enum RecalcWhen {\n DEFAULT = 0, // Calculate on new records or when any field in recalcDeps changes.\n NEVER = 1, // Don't calculate automatically (but user can trigger manually)\n MANUAL_UPDATES = 2, // Calculate on new records and on manual updates to any data field.\n}\n\n/**\n * Converts SQL type strings produced by the Sequelize library into its corresponding\n * Grist type. The list of types is based on an analysis of SQL type string outputs\n * produced by the Sequelize library (mostly covered in lib/data-types.js). Some\n * additional engine/dialect specific types are detailed in dialect directories.\n *\n * TODO: A handful of exotic SQL types (mostly from PostgreSQL) will currently throw an\n * Error, rather than returning a type. Further testing is required to determine\n * whether Grist can manage those data types.\n *\n * @param {String} sqlType A string produced by Sequelize's describeTable query\n * @return {String} The corresponding Grist type string\n * @throws {Error} If the sqlType is unrecognized or unsupported\n */\nexport function sequelizeToGristType(sqlType: string): GristType {\n // Sequelize type strings can include parens (e.g., `CHAR(10)`). This function\n // ignores those additional details when determining the Grist type.\n let endMarker = sqlType.length;\n const parensMarker = sqlType.indexOf(\"(\");\n endMarker = parensMarker > 0 ? parensMarker : endMarker;\n\n // Type strings might also include a space after the basic type description.\n // The type `DOUBLE PRECISION` is one such example, but modifiers or attributes\n // relevant to the type might also appear after the type itself (e.g., UNSIGNED,\n // NONZERO). These are ignored when determining the Grist type.\n const spaceMarker = sqlType.indexOf(\" \");\n endMarker = spaceMarker > 0 && spaceMarker < endMarker ? spaceMarker : endMarker;\n\n switch (sqlType.substring(0, endMarker)) {\n case \"INTEGER\":\n case \"BIGINT\":\n case \"SMALLINT\":\n case \"INT\":\n return \"Int\";\n case \"NUMBER\":\n case \"FLOAT\":\n case \"DECIMAL\":\n case \"NUMERIC\":\n case \"REAL\":\n case \"DOUBLE\":\n case \"DOUBLE PRECISION\":\n return \"Numeric\";\n case \"BOOLEAN\":\n case \"TINYINT\":\n return \"Bool\";\n case \"STRING\":\n case \"CHAR\":\n case \"TEXT\":\n case \"UUID\":\n case \"UUIDV1\":\n case \"UUIDV4\":\n case \"VARCHAR\":\n case \"NVARCHAR\":\n case \"TINYTEXT\":\n case \"MEDIUMTEXT\":\n case \"LONGTEXT\":\n case \"ENUM\":\n return \"Text\";\n case \"TIME\":\n case \"DATE\":\n case \"DATEONLY\":\n case \"DATETIME\":\n case \"NOW\":\n return \"Text\";\n case \"BLOB\":\n case \"TINYBLOB\":\n case \"MEDIUMBLOB\":\n case \"LONGBLOB\":\n // TODO: Passing binary data to the Sandbox is throwing Errors. Proper support\n // for these Blob data types requires some more investigation.\n throw new Error(\"SQL type: `\" + sqlType + \"` is currently unsupported\");\n case \"NONE\":\n case \"HSTORE\":\n case \"JSON\":\n case \"JSONB\":\n case \"VIRTUAL\":\n case \"ARRAY\":\n case \"RANGE\":\n case \"GEOMETRY\":\n throw new Error(\"SQL type: `\" + sqlType + \"` is currently untested\");\n default:\n throw new Error(\"Unrecognized datatype: `\" + sqlType + \"`\");\n }\n}\n\nexport function getReferencedTableId(type: string) {\n if (type === \"Attachments\") {\n return \"_grist_Attachments\";\n }\n return removePrefix(type, \"Ref:\") || removePrefix(type, \"RefList:\");\n}\n\nexport function isRefListType(type: string) {\n return type === \"Attachments\" || type?.startsWith(\"RefList:\");\n}\n\nexport function isListType(type: string) {\n return type === \"ChoiceList\" || isRefListType(type);\n}\n\nexport function isNumberType(type: string | undefined) {\n return [\"Numeric\", \"Int\"].includes(type || \"\");\n}\n\nexport function isDateLikeType(type: string) {\n return type === \"Date\" || type.startsWith(\"DateTime\");\n}\n\nexport function isFullReferencingType(type: string) {\n return type.startsWith(\"Ref:\") || isRefListType(type);\n}\n\nexport function isValidRuleValue(value: CellValue | undefined) {\n // We want to strictly test if a value is boolean, when the value is 0 or 1 it might\n // indicate other number in the future.\n return value === null || typeof value === \"boolean\";\n}\n\n/**\n * Returns true if `value` is blank.\n *\n * Blank values include `null`, (trimmed) empty string, and 0-length lists and\n * reference lists.\n */\nexport function isBlankValue(value: CellValue) {\n return (\n value === null ||\n (typeof value === \"string\" && value.trim().length === 0) ||\n isEmptyList(value) ||\n isEmptyReferenceList(value)\n );\n}\n\nexport type RefListValue = [GristObjCode.List, ...number[]] | null;\n\n/**\n * Type of cell metadata information.\n */\nexport enum CellInfoType {\n COMMENT = 1,\n}\n" }, { "path": "app/common/gristUrls.ts", "content": "import { AssistantConfig } from \"app/common/Assistant\";\nimport { BillingPage, BillingSubPage, BillingTask } from \"app/common/BillingAPI\";\nimport { OpenDocMode } from \"app/common/DocListAPI\";\nimport { EngineCode } from \"app/common/DocumentSettings\";\nimport { Features as PlanFeatures } from \"app/common/Features\";\nimport { encodeQueryParams, isAffirmative, removePrefix } from \"app/common/gutil\";\nimport { ICommonUrls } from \"app/common/ICommonUrls\";\nimport ICommonUrlsTI from \"app/common/ICommonUrls-ti\";\nimport { LocalPlugin } from \"app/common/plugin\";\nimport { StringUnion } from \"app/common/StringUnion\";\nimport { TelemetryLevel } from \"app/common/Telemetry\";\nimport { ThemeAppearance, themeAppearances, ThemeName, themeNames } from \"app/common/ThemePrefs\";\nimport { getGristConfig } from \"app/common/urlUtils\";\nimport { Document } from \"app/common/UserAPI\";\nimport { IAttachedCustomWidget } from \"app/common/widgetTypes\";\nimport { UIRowId } from \"app/plugin/GristAPI\";\n\nimport clone from \"lodash/clone\";\nimport pickBy from \"lodash/pickBy\";\nimport slugify from \"slugify\";\nimport * as t from \"ts-interface-checker\";\n\nconst { ICommonUrls: ICommonUrlsChecker } = t.createCheckers(ICommonUrlsTI);\n\nexport const SpecialDocPage = StringUnion(\n \"code\", \"acl\", \"data\", \"GristDocTour\",\n \"settings\", \"suggestions\", \"webhook\", \"timing\",\n);\ntype SpecialDocPage = typeof SpecialDocPage.type;\nexport type IDocPage = number | SpecialDocPage;\n\nexport type ViewDocPage = number | \"data\";\n/**\n * ViewDocPage is a page that shows table data (either normal or raw data view).\n */\nexport function isViewDocPage(docPage: IDocPage): docPage is ViewDocPage {\n return typeof docPage === \"number\" || docPage === \"data\";\n}\n\n// What page to show in the user's home area. Defaults to 'workspace' if a workspace is set, and\n// to 'all' otherwise.\nexport const HomePage = StringUnion(\"all\", \"workspace\", \"templates\", \"trash\");\nexport type IHomePage = typeof HomePage.type;\n\nexport const HomePageTab = StringUnion(\"recent\", \"pinned\", \"all\");\nexport type HomePageTab = typeof HomePageTab.type;\n\nexport const WelcomePage = StringUnion(\"teams\", \"signup\", \"verify\", \"select-account\");\nexport type WelcomePage = typeof WelcomePage.type;\n\nexport const AccountPage = StringUnion(\"account\");\nexport type AccountPage = typeof AccountPage.type;\n\nexport const ActivationPage = StringUnion(\"activation\");\nexport type ActivationPage = typeof ActivationPage.type;\n\nexport const AuditLogsPage = StringUnion(\"audit-logs\");\nexport type AuditLogsPage = typeof AuditLogsPage.type;\n\nexport const LoginPage = StringUnion(\"signup\", \"login\", \"verified\", \"forgot-password\");\nexport type LoginPage = typeof LoginPage.type;\n\nexport const AdminPanelPage = StringUnion(\"admin\", \"docs\", \"users\", \"workspaces\", \"orgs\");\nexport type AdminPanelPage = typeof AdminPanelPage.type;\n\nexport const AdminPanelTab = StringUnion(\"users\", \"workspaces\", \"docs\", \"orgs\", \"details\");\nexport type AdminPanelTab = typeof AdminPanelTab.type;\n\nexport const PREFERRED_STORAGE_ANCHOR = \"preferredStorage\";\nexport const PersistentAnchor = StringUnion(PREFERRED_STORAGE_ANCHOR);\nexport type PersistentAnchor = typeof PersistentAnchor.type;\n\n// Overall UI style. \"full\" is normal, \"singlePage\" is a single page focused, panels hidden experience.\nexport const InterfaceStyle = StringUnion(\"singlePage\", \"full\");\nexport type InterfaceStyle = typeof InterfaceStyle.type;\n\nexport const CompareEmphasis = StringUnion(\"local\", \"remote\");\nexport type CompareEmphasis = typeof CompareEmphasis.type;\n\n// Default subdomain for home api service if not otherwise specified.\nexport const DEFAULT_HOME_SUBDOMAIN = \"api\";\n\n// This is the minimum length a urlId may have if it is chosen\n// as a prefix of the docId.\nexport const MIN_URLID_PREFIX_LENGTH = 12;\n\n// Values meeting MIN_URLID_PREFIX_LENGTH that appear in non-document URLs and\n// should not be recognized as urlId prefixes when decoding URLs.\nconst RESERVED_URLID_PREFIXES = new Set([\"forgot-password\"]);\n\n// A prefix that identifies a urlId as a share key.\n// Important that this not be part of a valid docId.\nexport const SHARE_KEY_PREFIX = \"s.\";\n\n/**\n * Form framing is used to control the way forms are rendered in Grist.\n * - 'border' adds a green border around the form, used to indicate that the forms can be created\n * by untrusted users and it makes phishing attacks harder.\n * - 'minimal' doesn't show the border, used for trusted users.\n *\n * The default value is 'border', and it can be controlled by GRIST_FEATURE_FORM_FRAMING environment\n * variable.\n */\nexport type FormFraming = \"border\" | \"minimal\";\n\n/**\n * Special ways to open a document, based on what the user intends to do.\n * - view: Open document in read-only mode (even if user has edit rights)\n * - fork: Open document in fork-ready mode. This means that while edits are\n * permitted, those edits should go to a copy of the document rather than\n * the original.\n */\n\nexport const getCommonUrls = () => withAdminDefinedUrls({\n help: getHelpCenterUrl(),\n helpAccessRules: \"https://support.getgrist.com/access-rules\",\n helpAssistant: \"https://support.getgrist.com/assistant\",\n helpAssistantDataUse: \"https://support.getgrist.com/assistant/#data-use-policy\",\n helpFormulaAssistantDataUse: \"https://support.getgrist.com/ai-assistant/#data-use-policy\",\n helpColRefs: \"https://support.getgrist.com/col-refs\",\n helpConditionalFormatting: \"https://support.getgrist.com/conditional-formatting\",\n helpFilterButtons: \"https://support.getgrist.com/search-sort-filter/#pinning-filters\",\n helpLinkingWidgets: \"https://support.getgrist.com/linking-widgets\",\n helpRawData: \"https://support.getgrist.com/raw-data\",\n helpSuggestions: \"https://support.getgrist.com/sharing/#suggestions\",\n helpUnderstandingReferenceColumns: \"https://support.getgrist.com/col-refs/#understanding-reference-columns\",\n helpTriggerFormulas: \"https://support.getgrist.com/formulas/#trigger-formulas\",\n helpTryingOutChanges: \"https://support.getgrist.com/copying-docs/#trying-out-changes\",\n helpWidgets: \"https://support.getgrist.com/page-widgets/#widgets\",\n helpCustomWidgets: \"https://support.getgrist.com/widget-custom\",\n helpInstallAuditLogs: \"https://support.getgrist.com/install/audit-log-overview/\",\n helpTeamAuditLogs: \"https://support.getgrist.com/install/audit-log-overview/\",\n helpTelemetryLimited: \"https://support.getgrist.com/telemetry-limited\",\n helpEnterpriseOptIn: \"https://support.getgrist.com/self-managed/#how-do-i-enable-grist-enterprise\",\n helpCalendarWidget: \"https://support.getgrist.com/widget-calendar\",\n helpLinkKeys: \"https://support.getgrist.com/examples/2021-04-link-keys\",\n helpFilteringReferenceChoices: \"https://support.getgrist.com/col-refs/#filtering-reference-choices-in-dropdown-lists\",\n helpSandboxing: \"https://support.getgrist.com/self-managed/#how-do-i-sandbox-documents\",\n helpAPI: \"https://support.getgrist.com/api\",\n helpStateStore: \"https://support.getgrist.com/self-managed/#what-is-a-state-store\",\n helpSummaryFormulas: \"https://support.getgrist.com/summary-tables/#summary-formulas\",\n helpAdminControls: \"https://support.getgrist.com/admin-controls\",\n helpFiddleMode: \"https://support.getgrist.com/glossary/#fiddle-mode\",\n helpSharing: \"https://support.getgrist.com/sharing\",\n helpFormUrlValues: \"https://support.getgrist.com/widget-form/#accept-value-from-url\",\n helpAirtableIntegration: \"https://support.getgrist.com/install/integrations/airtable\",\n freeCoachingCall: getFreeCoachingCallUrl(),\n contactSupport: getContactSupportUrl(),\n termsOfService: getTermsOfServiceUrl(),\n onboardingTutorialVideoId: getOnboardingVideoId(),\n plans: \"https://www.getgrist.com/pricing\",\n contact: \"https://www.getgrist.com/contact\",\n templates: \"https://www.getgrist.com/templates\",\n webinars: getWebinarsUrl(),\n community: \"https://community.getgrist.com\",\n functions: \"https://support.getgrist.com/functions\",\n formulaSheet: \"https://support.getgrist.com/formula-cheat-sheet\",\n formulas: \"https://support.getgrist.com/formulas\",\n forms: \"https://www.getgrist.com/forms/?utm_source=grist-forms&utm_medium=grist-forms&utm_campaign=forms-footer\",\n openGraphPreviewImage: \"https://grist-static.com/icons/opengraph-preview-image.png\",\n\n gristLabsCustomWidgets: \"https://gristlabs.github.io/grist-widget/\",\n gristLabsWidgetRepository: \"https://github.com/gristlabs/grist-widget/releases/download/latest/manifest.json\",\n githubGristCore: \"https://github.com/gristlabs/grist-core\",\n githubSponsorGristLabs: \"https://github.com/sponsors/gristlabs\",\n\n versionCheck: \"https://api.getgrist.com/api/version\",\n attachmentStorage: \"https://support.getgrist.com/document-settings/#external-attachments\",\n\n signInWithGristRegister: \"https://login.getgrist.com/oauth/register\",\n signInWithGristHelp: \"https://support.getgrist.com/install/sign-in-with-grist\",\n});\n\nexport const commonUrls = getCommonUrls();\n\n/**\n * Values representable in a URL. The current state is available as urlState().state observable\n * in client. Updates to this state are expected by functions such as makeUrl() and setLinkUrl().\n */\nexport interface IGristUrlState {\n org?: string;\n homePage?: IHomePage;\n homePageTab?: HomePageTab;\n ws?: number;\n doc?: string;\n slug?: string; // if present, this is based on the document title, and is not a stable id\n mode?: OpenDocMode;\n fork?: UrlIdParts;\n docPage?: IDocPage;\n account?: AccountPage;\n billing?: BillingPage;\n activation?: ActivationPage;\n auditLogs?: AuditLogsPage;\n login?: LoginPage;\n welcome?: WelcomePage;\n adminPanel?: AdminPanelPage;\n adminPanelTab?: AdminPanelTab;\n welcomeTour?: boolean;\n docTour?: boolean;\n manageUsers?: boolean;\n createTeam?: boolean;\n upgradeTeam?: boolean;\n params?: {\n billingPlan?: string; // priceId\n planType?: string;\n billingTask?: BillingTask;\n embed?: boolean;\n state?: string;\n srcDocId?: string;\n style?: InterfaceStyle;\n compare?: string;\n compareEmphasis?: CompareEmphasis; // which of local and remote changes should be emphasized visually.\n linkParameters?: Record; // Parameters to pass as 'user.Link' in granular ACLs.\n // Encoded in URL as query params with extra '_' suffix.\n themeSyncWithOs?: boolean;\n themeAppearance?: ThemeAppearance;\n themeName?: ThemeName;\n details?: boolean; // Used on admin pages to show details tab.\n assistantPrompt?: string;\n assistantState?: string;\n };\n hash?: HashLink; // if present, this specifies an individual row within a section of a page.\n api?: boolean; // indicates that the URL should be encoded as an API URL, not as a landing page.\n // But this barely works, and is suitable only for documents. For decoding it\n // indicates that the URL probably points to an API endpoint.\n viaShare?: boolean; // Accessing document via a special share.\n form?: {\n vsId: number; // a view section id of a form.\n shareKey?: string; // only one of shareKey or doc should be set.\n },\n}\n\n// Subset of GristLoadConfig used by getOrgUrlInfo(), which affects the interpretation of the\n// current URL.\nexport interface OrgUrlOptions {\n // The org associated with the current URL.\n org?: string;\n\n // Base domain for constructing new URLs, should start with \".\" and not include port, e.g.\n // \".getgrist.com\". It should be unset for localhost operation and in single-org mode.\n baseDomain?: string;\n\n // In single-org mode, this is the single well-known org.\n singleOrg?: string;\n\n // Base URL used for accessing plugin material.\n pluginUrl?: string;\n\n // If set, org is expected to be encoded in the path, not domain.\n pathOnly?: boolean;\n}\n\n// Result of getOrgUrlInfo().\nexport interface OrgUrlInfo {\n hostname?: string; // If hostname should be changed to access the requested org.\n orgInPath?: string; // If /o/{orgInPath} should be used to access the requested org.\n}\n\nfunction hostMatchesUrl(host?: string, url?: string) {\n return host !== undefined && url !== undefined && new URL(url).host === host;\n}\n\n/**\n * Returns true if:\n * - the server is a home worker and the host matches APP_HOME_INTERNAL_URL;\n * - or the server is a doc worker and the host matches APP_DOC_INTERNAL_URL;\n *\n * @param {string?} host The host to check\n */\nfunction isOwnInternalUrlHost(host?: string) {\n // Note: APP_HOME_INTERNAL_URL may also be defined in doc worker as well as in home worker\n if (process.env.APP_HOME_INTERNAL_URL && hostMatchesUrl(host, process.env.APP_HOME_INTERNAL_URL)) {\n return true;\n }\n return Boolean(process.env.APP_DOC_INTERNAL_URL) && hostMatchesUrl(host, process.env.APP_DOC_INTERNAL_URL);\n}\n\n/**\n * Given host (optionally with port), baseDomain, and pluginUrl, determine whether to interpret host\n * as a custom domain, a native domain, or a plugin domain.\n */\nexport function getHostType(host: string, options: {\n baseDomain?: string, pluginUrl?: string\n}): \"native\" | \"custom\" | \"plugin\" {\n if (options.pluginUrl) {\n const url = new URL(options.pluginUrl);\n if (url.host.toLowerCase() === host.toLowerCase()) {\n return \"plugin\";\n }\n }\n\n const hostname = host.split(\":\")[0];\n if (!options.baseDomain) { return \"native\"; }\n if (\n hostname === \"localhost\" ||\n isOwnInternalUrlHost(host) ||\n hostname.endsWith(options.baseDomain)\n ) {\n return \"native\";\n }\n return \"custom\";\n}\n\nexport function getOrgUrlInfo(newOrg: string, currentHost: string, options: OrgUrlOptions): OrgUrlInfo {\n if (newOrg === options.singleOrg) {\n return {};\n }\n if (options.pathOnly) {\n return { orgInPath: newOrg };\n }\n const hostType = getHostType(currentHost, options);\n if (hostType !== \"plugin\") {\n const hostname = currentHost.split(\":\")[0];\n if (!options.baseDomain || hostname === \"localhost\") {\n return { orgInPath: newOrg };\n }\n }\n if (newOrg === options.org && hostType !== \"native\") {\n return {};\n }\n return { hostname: newOrg + options.baseDomain };\n}\n\n/**\n * The actual serialization of a url state into a URL. The URL has the form\n * /\n * /ws//\n * /doc/[/p/]\n *\n * where depends on whether subdomains are in use, e.g.\n * .getgrist.com\n * localhost:8080/o/\n */\nexport function encodeUrl(gristConfig: Partial,\n state: IGristUrlState, baseLocation: Location | URL,\n options: {\n tweaks?: UrlTweaks,\n } = {}): string {\n const url = new URL(baseLocation.href);\n const parts = [\"/\"];\n\n if (state.org) {\n // We figure out where to stick the org using the gristConfig and the current host.\n const { hostname, orgInPath } = getOrgUrlInfo(state.org, baseLocation.host, gristConfig);\n if (hostname) {\n url.hostname = hostname;\n }\n if (orgInPath) {\n parts.push(`o/${orgInPath}/`);\n }\n }\n\n if (state.api) {\n parts.push(`api/`);\n }\n if (state.ws) { parts.push(`ws/${state.ws}/`); }\n if (state.doc) {\n if (state.api) {\n parts.push(`docs/${encodeURIComponent(state.doc)}`);\n } else if (state.viaShare) {\n // Use a special path, and remove SHARE_KEY_PREFIX from id.\n let id = state.doc;\n if (id.startsWith(SHARE_KEY_PREFIX)) {\n id = id.substring(SHARE_KEY_PREFIX.length);\n }\n parts.push(`s/${encodeURIComponent(id)}`);\n } else if (state.slug) {\n parts.push(`${encodeURIComponent(state.doc)}/${encodeURIComponent(state.slug)}`);\n } else {\n parts.push(`doc/${encodeURIComponent(state.doc)}`);\n }\n if (state.mode && OpenDocMode.guard(state.mode)) {\n parts.push(`/m/${state.mode}`);\n }\n if (state.docPage) {\n parts.push(`/p/${state.docPage}`);\n }\n if (state.form) {\n parts.push(`/f/${state.form.vsId}`);\n }\n } else if (state.form?.shareKey) {\n parts.push(`forms/${encodeURIComponent(state.form.shareKey)}/${encodeURIComponent(state.form.vsId)}`);\n } else if (state.homePage === \"trash\" || state.homePage === \"templates\") {\n parts.push(`p/${state.homePage}`);\n }\n\n if (state.account) {\n parts.push(state.account === \"account\" ? \"account\" : `account/${state.account}`);\n }\n\n if (state.billing) {\n parts.push(state.billing === \"billing\" ? \"billing\" : `billing/${state.billing}`);\n }\n\n if (state.activation) { parts.push(state.activation); }\n\n if (state.auditLogs) { parts.push(state.auditLogs); }\n\n if (state.login) { parts.push(state.login); }\n\n if (state.welcome) {\n parts.push(`welcome/${state.welcome}`);\n }\n\n if (state.adminPanel) {\n parts.push(state.adminPanel === \"admin\" ? \"admin\" : `admin/${state.adminPanel}`);\n }\n\n const queryParams = pickBy(state.params, (v, k) => k !== \"linkParameters\") as { [key: string]: string };\n for (const [k, v] of Object.entries(state.params?.linkParameters || {})) {\n queryParams[`${k}_`] = v;\n }\n if (state.params?.details) {\n queryParams.details = \"true\";\n }\n const queryStr = encodeQueryParams(queryParams);\n\n url.pathname = parts.join(\"\");\n url.search = queryStr;\n\n if (state.homePageTab) {\n url.hash = state.homePageTab;\n } else if (state.hash?.anchor) {\n url.hash = state.hash.anchor;\n } else if (state.hash) {\n // Project tests use hashes, so only set hash if there is an anchor.\n url.hash = makeAnchorLinkValue(state.hash);\n } else if (state.welcomeTour) {\n url.hash = \"repeat-welcome-tour\";\n } else if (state.docTour) {\n url.hash = \"repeat-doc-tour\";\n } else if (state.manageUsers) {\n url.hash = \"manage-users\";\n } else if (state.createTeam) {\n url.hash = \"create-team\";\n } else if (state.upgradeTeam) {\n url.hash = \"upgrade-team\";\n } else if (state.adminPanelTab) {\n url.hash = state.adminPanelTab;\n } else {\n url.hash = \"\";\n }\n options.tweaks?.postEncode?.({\n url,\n parts,\n state,\n baseLocation,\n });\n return url.href;\n}\n\n/**\n * Parse a URL location into an IGristUrlState object. See encodeUrl() documentation.\n */\nexport function decodeUrl(gristConfig: Partial, location: Location | URL, options?: {\n tweaks?: UrlTweaks,\n}): IGristUrlState {\n location = new URL(location.href); // Make sure location is a URL.\n options?.tweaks?.preDecode?.({ url: location });\n const parts = location.pathname.slice(1).split(\"/\");\n const state: IGristUrlState = {};\n\n // Bare minimum we can do to detect API URLs: if it starts with /api/ or /o/{org}/api/...\n if (parts[0] === \"api\" || (parts[0] === \"o\" && parts[2] === \"api\")) {\n state.api = true;\n parts.splice(parts[0] === \"api\" ? 0 : 2, 1);\n }\n\n // Bare minimum we can do to detect form URLs with share keys: if it starts with /forms/ or /o/{org}/forms/...\n if (parts[0] === \"forms\" || (parts[0] === \"o\" && parts[2] === \"forms\")) {\n const startIndex = parts[0] === \"forms\" ? 0 : 2;\n // Form URLs have two parts to extract: the share key and the view section id.\n state.form = {\n shareKey: parts[startIndex + 1],\n vsId: parseInt(parts[startIndex + 2], 10),\n };\n parts.splice(startIndex, 3);\n }\n\n const map = new Map();\n for (let i = 0; i < parts.length; i += 2) {\n map.set(parts[i], decodeURIComponent(parts[i + 1]));\n }\n\n // For the API case, we need to map \"docs\" to \"doc\" (as this is what we did in encodeUrl and what API expects).\n if (state.api && map.has(\"docs\")) {\n map.set(\"doc\", map.get(\"docs\")!);\n }\n\n // /s/ is accepted as another way to write -> /doc/\n if (map.has(\"s\")) {\n const key = map.get(\"s\");\n map.set(\"doc\", `${SHARE_KEY_PREFIX}${key}`);\n state.viaShare = true;\n }\n\n // When the urlId is a prefix of the docId, documents are identified\n // as \"/slug\" instead of \"doc/\". We can detect that because\n // the minimum length of a urlId prefix is longer than the maximum length\n // of any of the valid keys in the url.\n for (const key of map.keys()) {\n if (\n key.length >= MIN_URLID_PREFIX_LENGTH &&\n !RESERVED_URLID_PREFIXES.has(key)\n ) {\n map.set(\"doc\", key);\n map.set(\"slug\", map.get(key)!);\n map.delete(key);\n break;\n }\n }\n\n const subdomain = parseSubdomain(location.host);\n if (gristConfig.org || gristConfig.singleOrg) {\n state.org = gristConfig.org || gristConfig.singleOrg;\n } else if (!gristConfig.pathOnly && subdomain.org) {\n state.org = subdomain.org;\n }\n const sp = new URLSearchParams(location.search);\n if (location.search) { state.params = {}; }\n if (map.has(\"o\")) { state.org = map.get(\"o\"); }\n if (map.has(\"ws\")) { state.ws = parseInt(map.get(\"ws\")!, 10); }\n if (map.has(\"doc\")) {\n state.doc = map.get(\"doc\");\n const fork = parseUrlId(map.get(\"doc\")!);\n if (fork.forkId) { state.fork = fork; }\n if (map.has(\"slug\")) { state.slug = map.get(\"slug\"); }\n if (map.has(\"p\")) { state.docPage = parseDocPage(map.get(\"p\")!); }\n if (map.has(\"f\")) { state.form = { vsId: parseInt(map.get(\"f\")!, 10) }; }\n } else {\n if (map.has(\"p\")) {\n const p = map.get(\"p\")!;\n state.homePage = HomePage.parse(p);\n }\n }\n if (map.has(\"m\")) { state.mode = OpenDocMode.parse(map.get(\"m\")); }\n if (map.has(\"account\")) { state.account = AccountPage.parse(map.get(\"account\")) || \"account\"; }\n if (map.has(\"billing\")) { state.billing = BillingSubPage.parse(map.get(\"billing\")) || \"billing\"; }\n if (map.has(\"activation\")) {\n state.activation = ActivationPage.parse(map.get(\"activation\")) || \"activation\";\n }\n if (map.has(\"audit-logs\")) {\n state.auditLogs = AuditLogsPage.parse(map.get(\"audit-logs\")) || \"audit-logs\";\n }\n if (map.has(\"welcome\")) { state.welcome = WelcomePage.parse(map.get(\"welcome\")); }\n if (map.has(\"admin\")) {\n state.adminPanel = AdminPanelPage.parse(map.get(\"admin\")) || \"admin\";\n }\n if (sp.has(\"planType\")) { state.params!.planType = sp.get(\"planType\")!; }\n if (sp.has(\"billingPlan\")) { state.params!.billingPlan = sp.get(\"billingPlan\")!; }\n if (sp.has(\"billingTask\")) {\n state.params!.billingTask = BillingTask.parse(sp.get(\"billingTask\"));\n }\n if (map.has(\"signup\")) {\n state.login = \"signup\";\n } else if (map.has(\"login\")) {\n state.login = \"login\";\n } else if (map.has(\"verified\")) {\n state.login = \"verified\";\n } else if (map.has(\"forgot-password\")) {\n state.login = \"forgot-password\";\n }\n if (sp.has(\"state\")) {\n state.params!.state = sp.get(\"state\")!;\n }\n if (sp.has(\"srcDocId\")) {\n state.params!.srcDocId = sp.get(\"srcDocId\")!;\n }\n if (sp.has(\"style\")) {\n let style = sp.get(\"style\");\n if (style === \"light\") {\n style = \"singlePage\";\n }\n\n state.params!.style = InterfaceStyle.parse(style);\n }\n if (sp.has(\"embed\")) {\n const embed = state.params!.embed = isAffirmative(sp.get(\"embed\"));\n // Turn view mode on if no mode has been specified, and not a fork.\n if (embed && !state.mode && !state.fork) { state.mode = \"view\"; }\n // Turn on single page style if no style has been specified.\n if (embed && !state.params!.style) { state.params!.style = \"singlePage\"; }\n }\n\n // Theme overrides\n if (sp.has(\"themeSyncWithOs\")) {\n state.params!.themeSyncWithOs = isAffirmative(sp.get(\"themeSyncWithOs\"));\n }\n\n if (sp.has(\"themeAppearance\")) {\n const appearance = sp.get(\"themeAppearance\");\n if (appearance && themeAppearances.includes(appearance as ThemeAppearance)) {\n state.params!.themeAppearance = appearance as ThemeAppearance;\n }\n }\n\n if (sp.has(\"themeName\")) {\n const themeName = sp.get(\"themeName\");\n if (themeName && themeNames.includes(themeName as ThemeName)) {\n state.params!.themeName = themeName as ThemeName;\n }\n }\n\n if (sp.has(\"details\")) {\n state.params!.details = isAffirmative(sp.get(\"details\"));\n }\n\n if (sp.has(\"compare\")) {\n state.params!.compare = sp.get(\"compare\")!;\n }\n if (sp.has(\"compareEmphasis\")) {\n const compareEmphasis = sp.get(\"compareEmphasis\")!;\n if (compareEmphasis && CompareEmphasis.guard(compareEmphasis)) {\n state.params!.compareEmphasis = compareEmphasis;\n }\n }\n const linkParameters = decodeLinkParameters(sp);\n if (linkParameters) {\n state.params!.linkParameters = linkParameters;\n }\n\n if (sp.has(\"assistantPrompt\")) {\n state.params!.assistantPrompt = sp.get(\"assistantPrompt\")!;\n }\n\n if (sp.has(\"assistantState\")) {\n state.params!.assistantState = sp.get(\"assistantState\")!;\n }\n\n if (location.hash) {\n const hash = location.hash;\n const hashParts = hash.split(\".\");\n const hashMap = new Map();\n for (const part of hashParts) {\n if (part.startsWith(\"rr\")) {\n hashMap.set(part.slice(0, 2), part.slice(2));\n } else {\n hashMap.set(part.slice(0, 1), part.slice(1));\n }\n }\n state.homePageTab = HomePageTab.parse(hashMap.get(\"#\"));\n state.adminPanelTab = AdminPanelTab.parse(hashMap.get(\"#\"));\n const anchor = PersistentAnchor.parse(hashMap.get(\"#\"));\n if (anchor) {\n state.hash = {\n anchor,\n };\n } else if (hashMap.has(\"#\") && [\"a1\", \"a2\", \"a3\", \"a4\"].includes(hashMap.get(\"#\") || \"\")) {\n const link: HashLink = {};\n const keys = [\n \"sectionId\",\n \"rowId\",\n \"colRef\",\n ] as (\"sectionId\" | \"rowId\" | \"colRef\")[];\n for (const key of keys) {\n let ch: string;\n if (key === \"rowId\" && hashMap.has(\"rr\")) {\n ch = \"rr\";\n link.rickRow = true;\n } else {\n ch = key.substr(0, 1);\n if (!hashMap.has(ch)) { continue; }\n }\n const value = hashMap.get(ch);\n if (key === \"rowId\" && value === \"new\") {\n link[key] = \"new\";\n } else if (key === \"rowId\" && value?.includes(\"-\")) {\n const rowIdParts = value.split(\"-\").map(p => (p === \"new\" ? p : parseInt(p, 10)));\n link[key] = rowIdParts[0];\n link.linkingRowIds = rowIdParts.slice(1);\n } else {\n link[key] = parseInt(value!, 10);\n }\n }\n if (hashMap.get(\"#\") === \"a2\") {\n link.popup = true;\n } else if (hashMap.get(\"#\") === \"a3\") {\n link.recordCard = true;\n } else if (hashMap.get(\"#\") === \"a4\") {\n link.comments = true;\n }\n state.hash = link;\n }\n state.welcomeTour = hashMap.get(\"#\") === \"repeat-welcome-tour\";\n state.docTour = hashMap.get(\"#\") === \"repeat-doc-tour\";\n state.manageUsers = hashMap.get(\"#\") === \"manage-users\";\n state.createTeam = hashMap.get(\"#\") === \"create-team\";\n state.upgradeTeam = hashMap.get(\"#\") === \"upgrade-team\";\n }\n return state;\n}\n\nexport function decodeLinkParameters(sp: URLSearchParams) {\n let linkParameters: Record | undefined = undefined;\n for (const [k, v] of sp.entries()) {\n if (k.endsWith(\"_\")) {\n if (!linkParameters) { linkParameters = {}; }\n linkParameters[k.slice(0, k.length - 1)] = v;\n }\n }\n return linkParameters;\n}\n\n// Returns a function suitable for user with makeUrl/setHref/etc, which updates aclAsUser*\n// linkParameters in the current state, unsetting them if email is null. Optional extraState\n// allows setting other properties (e.g. 'docPage') at the same time.\nexport function userOverrideParams(email: string | null, extraState?: IGristUrlState) {\n return function(prevState: IGristUrlState): IGristUrlState {\n const combined = { ...prevState, ...extraState };\n const linkParameters = clone(combined.params?.linkParameters) || {};\n if (email) {\n linkParameters.aclAsUser = email;\n } else {\n delete linkParameters.aclAsUser;\n }\n delete linkParameters.aclAsUserId;\n return { ...combined, params: { ...combined.params, linkParameters } };\n };\n}\n\n/**\n * parseDocPage is a noop for special pages, otherwise parse to integer\n */\nfunction parseDocPage(p: string): IDocPage {\n if (SpecialDocPage.guard(p)) {\n return p;\n }\n return parseInt(p, 10);\n}\n\n/**\n * Parses the URL like \"foo.bar.baz\" into the pair {org: \"foo\", base: \".bar.baz\"}.\n * Port is allowed and included into base.\n *\n * The \"base\" part is required to have at least two periods. The \"org\" part must pass\n * the subdomainRegex test.\n *\n * If there's no way to parse the URL into such a pair, then an empty object is returned.\n */\nexport function parseSubdomain(host: string | undefined): { org?: string, base?: string } {\n if (!host) { return {}; }\n const match = /^([^.]+)(\\..+\\..+)$/.exec(host.toLowerCase());\n if (match) {\n const org = match[1];\n const base = match[2];\n if (subdomainRegex.exec(org)) {\n return { org, base };\n }\n }\n // Host has nowhere to put a subdomain.\n return {};\n}\n\n// Allowed localhost addresses.\nconst localhostRegex = /^localhost(?::(\\d+))?$/i;\n\n/**\n * Like parseSubdomain, but throws an error if neither of these cases apply:\n * - host can be parsed into a valid subdomain and a valid base domain.\n * - host is localhost:NNNN\n * An empty object is only returned when host is localhost:NNNN.\n */\nexport function parseSubdomainStrictly(host: string | undefined): { org?: string, base?: string } {\n if (!host) { throw new Error(\"host not known\"); }\n const result = parseSubdomain(host);\n if (result.org) { return result; }\n if (!host.match(localhostRegex)) {\n throw new Error(`host not understood: ${host}`);\n }\n // Host is localhost[:NNNN], no org available.\n return {};\n}\n\n/**\n * For a packaged version of Grist that requires activation, this\n * summarizes the current state. Not applicable to grist-core.\n * This is the thing that is send via sendAppPage (so this is embedded in HTML).\n */\nexport interface ActivationState {\n installationId: string; // Unique identifier for this installation.\n key?: { // Set when Grist is activated.\n expirationDate?: string; // ISO8601 date that Grist will need reactivation.\n daysLeft?: number; // Number of days until Grist will need reactivation.\n },\n trial?: { // Present when installation has not yet been activated.\n days: number; // Max number of days allowed prior to activation.\n expirationDate: string; // ISO8601 date that Grist will get cranky.\n daysLeft: number; // Number of days left until Grist will get cranky.\n },\n needKey?: boolean; // Set when Grist is cranky and demanding activation.\n error?: string; // Present when there is an error reading the key.\n features?: PlanFeatures; // Features available in this installation.\n current?: Partial; // Usage of features in this installation.\n grace?: {\n daysLeft: number; // Number of days left in grace period.\n graceStarted: string; // ISO8601 date when grace period started.\n }\n}\n\nexport interface LatestVersionAvailable {\n version: string;\n isNewer: boolean;\n isCritical: boolean;\n dateChecked: number;\n releaseUrl?: string;\n}\n\n/**\n * These settings get sent to the client along with the loaded page. At the minimum, the browser\n * needs to know the URL of the home API server (e.g. api.getgrist.com).\n */\nexport interface GristLoadConfig {\n // URL of the Home API server for the browser client to use.\n homeUrl: string | null;\n\n // When loading /doc/{docId}, we include the id used to assign the document (this is the docId).\n assignmentId?: string;\n\n // Org or \"subdomain\". When present, this overrides org information from the hostname. We rely\n // on this for custom domains, but set it generally for all pages.\n org?: string;\n\n // Makes the Grist frontend access the Grist instance using its current URL in the browser, rather than APP_HOME_URL.\n // Used to simplify setup of single-domain (no subdomain / doc worker) installations.\n serveSameOrigin?: boolean;\n\n // Base domain for constructing new URLs, should start with \".\" and not include port, e.g.\n // \".getgrist.com\". It should be unset for localhost operation and in single-org mode.\n baseDomain?: string;\n\n // In single-org mode, this is the single well-known org. Suppress any org selection UI.\n singleOrg?: string;\n\n // Url for support for the browser client to use.\n helpCenterUrl?: string;\n\n // Url for terms of service for the browser client to use\n termsOfServiceUrl?: string;\n\n // Url for free coaching call scheduling for the browser client to use.\n freeCoachingCallUrl?: string;\n\n // Url for \"contact support\" button on Grist's \"not found\" error page\n contactSupportUrl?: string;\n\n // Url for webinars.\n webinarsUrl?: string;\n\n // When set, this directs the client to encode org information in path, not in domain.\n pathOnly?: boolean;\n\n // Type of error page to show. This is used for pages such as \"signed-out\" and \"not-found\",\n // which don't include the full app.\n errPage?: string;\n\n // When errPage is a generic \"other-error\", this is the message to show.\n errMessage?: string;\n\n // When errPage is a generic page, not an error, this is additional details to show.\n errDetails?: Record;\n\n // When an error page is shown in response to a request for an URL, this is the URL that was\n // originally requested — this may not be the URL we're responding to, because of an\n // intermediate redirect in case of e.g. an OIDC sign-in.\n // The error page will set the browser's current URL to that, so that the user can\n // retry by simply refreshing the page.\n errTargetUrl?: string;\n\n // URL for client to use for untrusted content.\n pluginUrl?: string;\n\n // Stripe API key for use on the client.\n stripeAPIKey?: string;\n\n // BeaconID for the support widget from HelpScout.\n helpScoutBeaconId?: string;\n\n // If set, enable anonymous sharing UI elements.\n supportAnon?: boolean;\n\n // If set, enable anonymous playground.\n enableAnonPlayground?: boolean;\n\n // If set, allow non-admins to create new organizations\n canAnyoneCreateOrgs?: boolean;\n\n // If set, allow access to each user's personal organization\n enablePersonalOrgs?: boolean;\n\n // If set, allow selection of the specified engines.\n // TODO: move this list to a separate endpoint.\n supportEngines?: EngineCode[];\n\n // Max upload allowed for imports (except .grist files), in bytes; 0 or omitted for unlimited.\n maxUploadSizeImport?: number;\n\n // Max upload allowed for attachments, in bytes; 0 or omitted for unlimited.\n maxUploadSizeAttachment?: number;\n\n // Pre-fetched call to getDoc for the doc being loaded.\n getDoc?: { [id: string]: Document };\n\n // Pre-fetched call to getWorker for the doc being loaded.\n getWorker?: { [id: string]: string | null };\n\n // The timestamp when this gristConfig was generated.\n timestampMs: number;\n\n // Google Client Id, used in Google integration (ex: Google Drive Plugin)\n googleClientId?: string;\n\n // Max scope we can request for accessing files from Google Drive.\n // Default used by Grist is https://www.googleapis.com/auth/drive.file:\n // View and manage Google Drive files and folders that you have opened or created with this app.\n // More on scopes: https://developers.google.com/identity/protocols/oauth2/scopes#drive\n googleDriveScope?: string;\n\n // List of registered plugins (used by HomePluginManager and DocPluginManager)\n plugins?: LocalPlugin[];\n\n // If additional custom widgets (besides the Custom URL widget) should be shown in\n // the custom widget gallery.\n enableWidgetRepository?: boolean;\n\n // Whether there is somewhere for survey data to go.\n survey?: boolean;\n\n // Google Tag Manager id. Currently only used to load tag manager for reporting new sign-ups.\n tagManagerId?: string;\n\n activation?: ActivationState;\n\n // Latest Grist release available\n latestVersionAvailable?: LatestVersionAvailable;\n\n // Is automatic version checking allowed?\n automaticVersionCheckingAllowed?: boolean;\n\n // List of enabled features.\n features?: IFeature[];\n\n // String to append to the end of the HTML document.title\n pageTitleSuffix?: string;\n\n // If custom CSS should be included in the head of each page.\n enableCustomCss?: boolean;\n\n // Supported languages for the UI. By default only english (en) is supported.\n supportedLngs?: readonly string[];\n\n // Loaded namespaces for translations.\n namespaces?: readonly string[];\n\n assistant?: AssistantConfig;\n\n permittedCustomWidgets?: IAttachedCustomWidget[];\n\n // Email address of the support user.\n supportEmail?: string;\n\n // Current user locale, read from the user options;\n userLocale?: string;\n\n // Telemetry config.\n telemetry?: TelemetryConfig;\n\n // The Grist deployment type (e.g. core, enterprise).\n deploymentType?: GristDeploymentType;\n\n // Force enterprise deployment? For backwards compatibility with grist-ee Docker image\n forceEnableEnterprise?: boolean;\n\n // The org containing public templates and tutorials.\n templateOrg?: string | null;\n\n // The doc id of the tutorial shown during onboarding.\n onboardingTutorialDocId?: string;\n\n // The id of the Youtube video to show for the onboarding\n onboardingTutorialVideoId?: string;\n\n // Whether to show the \"Delete Account\" button in the account page.\n canCloseAccount?: boolean;\n\n experimentalPlugins?: boolean;\n\n // If backend has an email service for sending notifications.\n notifierEnabled?: boolean;\n\n // Set on /admin pages only, when AdminControls are available and should be enabled in UI.\n adminControls?: boolean;\n\n formFraming?: FormFraming;\n\n adminDefinedUrls?: string;\n\n // Maximum users to display for user presence features (e.g. active user list)\n userPresenceMaxUsers?: number;\n\n warnBeforeSharingPublicly?: boolean;\n\n // Whether there is a parent process that can restart Grist.\n runningUnderSupervisor?: boolean;\n}\n\nexport const Features = StringUnion(\n \"helpCenter\",\n \"billing\",\n \"templates\",\n \"createSite\",\n \"multiSite\",\n \"multiAccounts\",\n \"importFromAirtable\",\n \"sendToDrive\",\n \"tutorials\",\n \"supportGrist\",\n \"themes\",\n);\nexport type IFeature = typeof Features.type;\n\n// Features that are enabled, even if not explicitly listed in GRIST_UI_FEATURES.\n// These should be still be disabled if listed in GRIST_HIDE_UI_ELEMENTS.\nexport const ImplicitlyEnabledFeatures: IFeature[] = [\"importFromAirtable\"];\n\nexport function isFeatureEnabled(feature: IFeature): boolean {\n return (getGristConfig().features || []).includes(feature);\n}\n\nexport function getPageTitleSuffix(config?: GristLoadConfig) {\n return config?.pageTitleSuffix ?? \" - Grist\";\n}\n\nexport interface TelemetryConfig {\n telemetryLevel: TelemetryLevel;\n}\n\nexport const GristDeploymentTypes = StringUnion(\"saas\", \"core\", \"enterprise\", \"electron\", \"static\");\nexport type GristDeploymentType = typeof GristDeploymentTypes.type;\n\n// Acceptable org subdomains are alphanumeric (hyphen also allowed) and of\n// non-zero length.\nconst subdomainRegex = /^[-a-z0-9]+$/i;\n\nexport interface OrgParts {\n subdomain: string | null;\n orgFromHost: string | null;\n orgFromPath: string | null;\n pathRemainder: string;\n mismatch: boolean;\n}\n\n/**\n * Returns true if code is running in client, false if running in server.\n */\nexport function isClient() {\n return (typeof window !== \"undefined\") && window && window.location?.hostname;\n}\n\nfunction getCustomizableValue(\n clientSideConfigKey: keyof GristLoadConfig,\n serverSideEnvVar: keyof NodeJS.ProcessEnv,\n) {\n return isClient() ? (window as any).gristConfig?.[clientSideConfigKey] : process.env[serverSideEnvVar];\n}\n\n/**\n * Returns a known org \"subdomain\" if Grist is configured in single-org mode\n * (GRIST_SINGLE_ORG= on the server) or if the page includes an org in gristConfig.\n */\nexport function getSingleOrg(): string | null {\n return getCustomizableValue(\"singleOrg\", \"GRIST_SINGLE_ORG\") || null;\n}\n\nexport function getHelpCenterUrl(): string {\n const defaultUrl = \"https://support.getgrist.com\";\n return getCustomizableValue(\"helpCenterUrl\", \"GRIST_HELP_CENTER\") || defaultUrl;\n}\n\nexport function getOnboardingVideoId(): string {\n const defaultId = \"56AieR9rpww\";\n return getCustomizableValue(\"onboardingTutorialVideoId\", \"GRIST_ONBOARDING_VIDEO_ID\") || defaultId;\n}\n\nexport function getTermsOfServiceUrl(): string | undefined {\n return getCustomizableValue(\"termsOfServiceUrl\", \"GRIST_TERMS_OF_SERVICE_URL\") || undefined;\n}\n\nexport function getFreeCoachingCallUrl(): string {\n const defaultUrl = \"https://calendly.com/grist-team/grist-free-coaching-call\";\n return getCustomizableValue(\"freeCoachingCallUrl\", \"FREE_COACHING_CALL_URL\") || defaultUrl;\n}\n\nexport function getContactSupportUrl(): string {\n const defaultUrl = \"https://www.getgrist.com/contact\";\n return getCustomizableValue(\"contactSupportUrl\", \"GRIST_CONTACT_SUPPORT_URL\") || defaultUrl;\n}\n\nexport function getWebinarsUrl(): string {\n const defaultUrl = \"https://www.getgrist.com/webinars/grist-101-new-users-guide\";\n return getCustomizableValue(\"webinarsUrl\", \"GRIST_WEBINARS_URL\") || defaultUrl;\n}\n\nexport function getMaxUploadSizeAttachmentMB(): number {\n const value = getCustomizableValue(\"maxUploadSizeAttachment\", \"GRIST_MAX_UPLOAD_ATTACHMENT_MB\");\n return Number(value) || Infinity;\n}\n\n/**\n * Returns true if org must be encoded in path, not in domain. Determined from\n * gristConfig on the client. On the server, returns true if the host is\n * supplied and is 'localhost', or if GRIST_ORG_IN_PATH is set to 'true'.\n */\nexport function isOrgInPathOnly(host?: string): boolean {\n if (isClient()) {\n const gristConfig: GristLoadConfig = (window as any).gristConfig;\n return (gristConfig?.pathOnly) || false;\n } else {\n if (host?.match(localhostRegex)) { return true; }\n return (process.env.GRIST_ORG_IN_PATH === \"true\");\n }\n}\n\n// Extract an organization name from the host. Returns null if an organization name\n// could not be recovered. Organization name may be overridden by server configuration.\nexport function getOrgFromHost(reqHost: string): string | null {\n const singleOrg = getSingleOrg();\n if (singleOrg) { return singleOrg; }\n if (isOrgInPathOnly()) { return null; }\n return parseSubdomain(reqHost).org || null;\n}\n\n/**\n * Get any information about an organization that is embedded in the host name or the\n * path.\n * For example, on nasa.getgrist.com, orgFromHost and subdomain will be set to \"nasa\".\n * On localhost:8000/o/nasa, orgFromPath and subdomain will be set to \"nasa\".\n * On nasa.getgrist.com/o/nasa, orgFromHost, orgFromPath, and subdomain will all be \"nasa\".\n * On spam.getgrist.com/o/nasa, orgFromHost will be \"spam\", orgFromPath will be \"nasa\",\n * subdomain will be null, and mismatch will be true.\n */\nexport function extractOrgParts(reqHost: string | undefined, reqPath: string): OrgParts {\n let orgFromHost: string | null = getSingleOrg();\n\n if (!orgFromHost && reqHost) {\n orgFromHost = getOrgFromHost(reqHost);\n if (orgFromHost) {\n // Some subdomains are shared, and do not reflect the name of an organization.\n // See /documentation/urls.md for a list.\n if (/^(api|v1-.*|doc-worker-.*)$/.test(orgFromHost)) {\n orgFromHost = null;\n }\n }\n }\n\n const part = parseFirstUrlPart(\"o\", reqPath);\n if (part.value) {\n const orgFromPath = part.value.toLowerCase();\n const mismatch = Boolean(orgFromHost && orgFromPath && (orgFromHost !== orgFromPath));\n const subdomain = mismatch ? null : orgFromPath;\n return { orgFromHost, orgFromPath, pathRemainder: part.path, mismatch, subdomain };\n }\n return { orgFromHost, orgFromPath: null, pathRemainder: reqPath, mismatch: false, subdomain: orgFromHost };\n}\n\n/**\n * When a prefix is extracted from the path, the remainder of the path may be empty.\n * This method makes sure there is at least a \"/\".\n */\nexport function sanitizePathTail(path: string | undefined) {\n path = path || \"/\";\n return (path.startsWith(\"/\") ? \"\" : \"/\") + path;\n}\n\n/*\n * If path starts with /{tag}/{value}{/rest}, returns value and the remaining path (/rest).\n * Otherwise, returns value of undefined and the path unchanged.\n * E.g. parseFirstUrlPart('o', '/o/foo/bar') returns {value: 'foo', path: '/bar'}.\n */\nexport function parseFirstUrlPart(tag: string, path: string): { value?: string, path: string } {\n const match = path.match(/^\\/([^/?#]+)\\/([^/?#]+)(.*)$/);\n if (match?.[1] === tag) {\n return { value: match[2], path: sanitizePathTail(match[3]) };\n } else {\n return { path };\n }\n}\n\n/**\n * The internal structure of a UrlId. There is no internal structure,\n * except in the following cases. The id may be for a fork, in which\n * case the fork has a separate id, and a user id may also be embedded\n * to track ownership. The id may be a share key, in which case it\n * has some special syntax to identify it as so.\n */\nexport interface UrlIdParts {\n trunkId: string;\n forkId?: string;\n forkUserId?: number;\n snapshotId?: string;\n shareKey?: string;\n}\n\n// Parse a string of the form trunkId or trunkId~forkId or trunkId~forkId~forkUserId\n// or trunkId[....]~v=snapshotId\n// or shareKey\nexport function parseUrlId(urlId: string): UrlIdParts {\n let snapshotId: string | undefined;\n const parts = urlId.split(\"~\");\n const bareParts = parts.filter(part => !part.includes(\"v=\"));\n for (const part of parts) {\n if (part.startsWith(\"v=\")) {\n snapshotId = decodeURIComponent(part.substr(2).replace(/_/g, \"%\"));\n }\n }\n const trunkId = bareParts[0];\n // IDs starting with SHARE_KEY_PREFIX are in fact shares.\n const shareKey = removePrefix(trunkId, SHARE_KEY_PREFIX) || undefined;\n return {\n trunkId: bareParts[0],\n forkId: bareParts[1],\n forkUserId: (bareParts[2] !== undefined) ? parseInt(bareParts[2], 10) : undefined,\n snapshotId,\n shareKey,\n };\n}\n\n// Construct a string of the form trunkId or trunkId~forkId or trunkId~forkId~forkUserId\n// or trunkId[....]~v=snapshotId\nexport function buildUrlId(parts: UrlIdParts): string {\n let token = [parts.trunkId, parts.forkId, parts.forkUserId].filter(x => x !== undefined).join(\"~\");\n if (parts.snapshotId) {\n // This could be an S3 VersionId, about which AWS makes few promises.\n // encodeURIComponent leaves untouched the following:\n // alphabetic; decimal; any of: - _ . ! ~ * ' ( )\n // We further encode _.!~*'() to fit within existing limits on what characters\n // may be in a docId (leaving just the hyphen, which is permitted). The limits\n // could be loosened, but without much benefit.\n const codedSnapshotId = encodeURIComponent(parts.snapshotId)\n .replace(/[_.!~*'()-]/g, ch => `_${ch.charCodeAt(0).toString(16).toUpperCase()}`)\n .replace(/%/g, \"_\");\n token = `${token}~v=${codedSnapshotId}`;\n }\n return token;\n}\n\n/**\n * Values that may be encoded in a hash in a document url.\n */\nexport interface HashLink {\n sectionId?: number;\n rowId?: UIRowId;\n colRef?: number;\n popup?: boolean;\n comments?: boolean; // Whether to show comments in the popup.\n rickRow?: boolean;\n recordCard?: boolean;\n linkingRowIds?: UIRowId[];\n anchor?: string;\n}\n\n/**\n * Encode a HashLink as a string to include as the URL fragment (hash prooperty). For example,\n * in https://templates.getgrist.com/doc/lightweight-crm#a1.s1.r7.c2, the \"a1.s1.r7.c2\" portion is\n * the anchor link. The parts have the following meaning:\n * a = identifies the type of link (1 is normal, 2 for popup, 3 for record-card)\n * s = sectionId (rowId of the page-widget)\n * r = rowId (of the actual row in the user table)\n * c = colRef (rowId of the column's metadata record)\n */\nexport function makeAnchorLinkValue(hash: HashLink): string {\n const hashParts: string[] = [];\n if (hash.rowId || hash.popup || hash.recordCard) {\n if (hash.comments) {\n hashParts.push(\"a4\");\n } else if (hash.recordCard) {\n hashParts.push(\"a3\");\n } else if (hash.popup) {\n hashParts.push(\"a2\");\n } else if (hash.anchor) {\n hashParts.push(hash.anchor);\n } else {\n hashParts.push(\"a1\");\n }\n for (const key of [\"sectionId\", \"rowId\", \"colRef\"] as (keyof HashLink)[]) {\n let enhancedRowId: string | undefined;\n if (key === \"rowId\" && hash.linkingRowIds?.length) {\n enhancedRowId = [hash.rowId, ...hash.linkingRowIds].join(\"-\");\n }\n const partValue = enhancedRowId ?? hash[key];\n if (partValue) {\n const partKey = key === \"rowId\" && hash.rickRow ? \"rr\" : key[0];\n hashParts.push(`${partKey}${partValue}`);\n }\n }\n }\n return hashParts.join(\".\");\n}\n\n// Check whether a urlId is a prefix of the docId, and adequately long to be\n// a candidate for use in prettier urls.\nfunction shouldIncludeSlug(doc: { id: string, urlId: string | null }): boolean {\n if (!doc.urlId || doc.urlId.length < MIN_URLID_PREFIX_LENGTH) { return false; }\n return doc.id.startsWith(doc.urlId) || doc.urlId.startsWith(SHARE_KEY_PREFIX);\n}\n\n// Convert the name of a document into a slug. The slugify library normalizes unicode characters,\n// replaces those with a reasonable ascii representation. Only alphanumerics are retained, and\n// spaces are replaced with hyphens.\nfunction nameToSlug(name: string): string {\n return slugify(name, { strict: true });\n}\n\n// Returns a slug for the given docId/urlId/name, or undefined if a slug should\n// not be used.\nexport function getSlugIfNeeded(doc: { id: string, urlId: string | null, name: string }): string | undefined {\n if (!shouldIncludeSlug(doc)) { return; }\n return nameToSlug(doc.name);\n}\n\n/**\n * It is possible we want to remap Grist URLs in some way - specifically,\n * grist-static does this. We allow for a hook that is called after\n * encoding state as a URL, and a hook that is called before decoding\n * state from a URL.\n */\nexport interface UrlTweaks {\n /**\n * Tweak an encoded URL. Operates on the URL directly, in place.\n */\n postEncode?(options: {\n url: URL,\n parts: string[],\n state: IGristUrlState,\n baseLocation: Location | URL,\n }): void;\n\n /**\n * Tweak a URL prior to decoding it. Operates on the URL directly, in place.\n */\n preDecode?(options: {\n url: URL,\n }): void;\n}\n\nfunction withAdminDefinedUrls(defaultUrls: ICommonUrls): ICommonUrls {\n const adminDefinedUrlsStr = getCustomizableValue(\"adminDefinedUrls\", \"GRIST_CUSTOM_COMMON_URLS\");\n if (!adminDefinedUrlsStr) {\n return defaultUrls;\n }\n\n let adminDefinedUrls;\n try {\n adminDefinedUrls = JSON.parse(adminDefinedUrlsStr);\n } catch (e) {\n throw new Error(\"The JSON passed to GRIST_CUSTOM_COMMON_URLS is malformed\");\n }\n\n const merged = {\n ...defaultUrls,\n ...(adminDefinedUrls),\n };\n ICommonUrlsChecker.strictCheck(merged);\n return merged;\n}\n" }, { "path": "app/common/gutil.ts", "content": "import {\n BindableValue,\n Computed,\n DomElementMethod,\n Holder,\n IDisposableOwner,\n IKnockoutReadObservable,\n ISubscribable,\n Listener,\n MultiHolder,\n Observable,\n subscribeElem,\n UseCB,\n UseCBOwner,\n} from \"grainjs\";\nimport { Observable as KoObservable } from \"knockout\";\nimport identity from \"lodash/identity\";\n\n// Some definitions have moved to be used by plugin API.\nexport { arrayRepeat } from \"app/plugin/gutil\";\n\nexport const UP_TRIANGLE = \"\\u25B2\";\nexport const DOWN_TRIANGLE = \"\\u25BC\";\n\nconst EMAIL_RE = new RegExp(\"^\\\\w[\\\\w%+/='-]*(\\\\.[\\\\w%+/='-]+)*@([A-Za-z0-9]([A-Za-z0-9-]*[A-Za-z\" +\n \"0-9])?\\\\.)+[A-Za-z]{2,24}$\", \"u\");\n\n// Returns whether str starts with prefix. (Note that this implementation avoids creating a new\n// string, and only checks a single location.)\nexport function startsWith(str: string, prefix: string): boolean {\n return str.lastIndexOf(prefix, 0) === 0;\n}\n\n// Returns whether str ends with suffix.\nexport function endsWith(str: string, suffix: string): boolean {\n return str.includes(suffix, str.length - suffix.length);\n}\n\n// If str starts with prefix, removes it and returns what remains. Otherwise, returns null.\nexport function removePrefix(str: string, prefix: string): string | null {\n return startsWith(str, prefix) ? str.slice(prefix.length) : null;\n}\n\n// If str ends with suffix, removes it and returns what remains. Otherwise, returns null.\nexport function removeSuffix(str: string, suffix: string): string | null {\n return endsWith(str, suffix) ? str.slice(0, str.length - suffix.length) : null;\n}\n\nexport function removeTrailingSlash(str: string): string {\n const result = removeSuffix(str, \"/\");\n return result === null ? str : result;\n}\n\n// Expose .padStart. The version of node we use has it, but they typings\n// need the es2017 typescript target. TODO: replace once typings in place.\nexport function padStart(str: string, targetLength: number, padString: string) {\n return (str as any).padStart(targetLength, padString);\n}\n\n// Capitalizes every word in a string.\nexport function capitalize(str: string): string {\n return str.replace(/\\b[a-z]/gi, c => c.toUpperCase());\n}\n\n// Capitalizes the first word in a string.\nexport function capitalizeFirstWord(str: string): string {\n return str.replace(/\\b[a-z]/i, c => c.toUpperCase());\n}\n\n// Returns whether the string n represents a valid number.\n// http://stackoverflow.com/questions/18082/validate-numbers-in-javascript-isnumeric\nexport function isNumber(n: string): boolean {\n // This wasn't right for a long time: isFinite() is key to failing on strings like \"5a\".\n return !isNaN(parseFloat(n)) && isFinite(n as any);\n}\n\n/**\n * Returns a value clamped to the given min-max range.\n * @param {Number} value - some numeric value.\n * @param {Number} min - minimum value allowed.\n * @param {Number} max - maximum value allowed. Must have min <= max.\n * @returns {Number} - value restricted to the given range.\n */\nexport function clamp(value: number, min: number, max: number): number {\n return Math.max(min, Math.min(max, value));\n}\n\n/**\n * Checks if ele is contained within the given bounds.\n * @param {Number} value\n * @param {Number} bound1 - does not have to be less than/equal to bound2\n * @param {Number} bound2\n * @returns {Boolean} - True/False\n */\nexport function between(value: number, bound1: number, bound2: number): boolean {\n const lower = Math.min(bound1, bound2);\n const upper = Math.max(bound1, bound2);\n return lower <= value && value <= upper;\n}\n\n/**\n * Returns the positive modulo of x by n. (Javascript default allows negatives)\n */\nexport function mod(x: number, n: number): number {\n return ((x % n) + n) % n;\n}\n\n/**\n * Returns a number that is n rounded down to the next nearest number divisible by m\n */\n\nexport function roundDownToMultiple(n: number, m: number): number {\n return Math.floor(n / m) * m;\n}\n\n/**\n * Returns the first argument unless it's undefined, in which case returns the second one.\n */\nexport function undefDefault(x: T | undefined, y: T): T {\n return (x !== void 0) ? x : y;\n}\n\n// for typescript 4\n// type Undef = T extends [infer A, ...infer B] ? undefined extends A ? NonNullable | Undef : A : unknown;\n\ntype Undef1 = T extends [infer A] ? A : unknown;\n\ntype Undef2 = T extends [infer A, infer B] ?\n undefined extends A ? NonNullable | Undef1<[B]> : A : Undef1;\n\ntype Undef3 = T extends [infer A, infer B, infer C] ?\n undefined extends A ? NonNullable | Undef2<[B, C]> : A : Undef2;\n\ntype Undef = T extends [infer A, infer B, infer C, infer D] ?\n undefined extends A ? NonNullable | Undef3<[B, C, D]> : A : Undef3;\n\n/*\n\nUndef can detect correct type that will be returned as a first defined value:\n\nconst t1: number = undef(1, 1 as number | undefined);\nconst t1: number | undefined = undef(2 as number | undefined, 3 as number | undefined);\nconst t3: number = undef(3 as number | undefined, undefined, 4);\nconst t4: number = undef(1, '');\nconst t5: number = undef(1 as number | undefined, 4);\nconst t6: string = undef('1', 2);\nconst t7: string | number = undef(undefined, 2 as number | undefined, '3');\nconst t8: string = undef(undefined, undefined, '3');\nconst t9: string = undef(undefined, '2' as string | undefined, '3');\nconst ta: string | number | undefined = undef(undefined, '2' as string | undefined, 3 as number | undefined);\nconst tb: string | number = undef(undefined, '2' as string | undefined, 3 as number | undefined, 5);\n*/\n\n/**\n * Returns the first defined value from the list or unknown.\n * Use with typed result, so the typescript type checker can provide correct type.\n */\nexport function undef(...list: T): Undef {\n for (const value of list) {\n if (value !== undefined) { return value; }\n }\n return undefined as any;\n}\n\n/**\n * Returns the number representation of `value`, or `defaultVal` if it cannot\n * be represented as a valid number.\n */\nexport function numberOrDefault(value: unknown, defaultVal: T): number | T {\n if (typeof value === \"number\") {\n return !Number.isNaN(value) ? value : defaultVal;\n } else if (typeof value === \"string\") {\n const maybeNumber = Number.parseFloat(value);\n return !Number.isNaN(maybeNumber) ? maybeNumber : defaultVal;\n } else {\n return defaultVal;\n }\n}\n\n/**\n * Parses json and returns the result, or returns defaultVal if parsing fails.\n */\nexport function safeJsonParse(json: string, defaultVal: any): any {\n try {\n return json !== \"\" && json !== undefined ? JSON.parse(json) : defaultVal;\n } catch (e) {\n return defaultVal;\n }\n}\n\n/**\n * Just like encodeURIComponent, but does not encode slashes. Slashes don't hurt to be included in\n * URL parameters, and look much friendlier not encoded.\n */\nexport function encodeQueryParam(str: string | number | undefined): string {\n return encodeURIComponent(String(str === undefined ? null : str)).replace(/%2F/g, \"/\");\n}\n\n/**\n * Encode an object into a querystring (\"key=value&key2=value2\").\n * This is similar to JQuery's $.param, but only works on shallow objects.\n */\nexport function encodeQueryParams(obj: { [key: string]: string | number | undefined }): string {\n return Object.keys(obj).map((k: string) => encodeQueryParam(k) + \"=\" + encodeQueryParam(obj[k])).join(\"&\");\n}\n\n/**\n * Return a list of the words in the string, using the given separator string. At most\n * maxNumSplits splits are done, so the result will have at most maxNumSplits + 1 elements (this\n * is the main difference from how JS built-in string.split() works, and similar to Python split).\n * @param {String} str: String to split.\n * @param {String} sep: Separator to split on.\n * @param {Number} maxNumSplits: Maximum number of splits to do.\n * @return {Array[String]} Array of words, of length at most maxNumSplits + 1.\n */\nexport function maxsplit(str: string, sep: string, maxNumSplits: number): string[] {\n const result: string[] = [];\n let start = 0, pos;\n for (let i = 0; i < maxNumSplits; i++) {\n pos = str.indexOf(sep, start);\n if (pos === -1) {\n break;\n }\n result.push(str.slice(start, pos));\n start = pos + sep.length;\n }\n result.push(str.slice(start));\n return result;\n}\n\n// Compare arrays of scalars for equality.\nexport function arraysEqual(a: any[], b: any[]): boolean {\n if (a === b) {\n return true;\n }\n if (!a || !b) {\n return false;\n }\n if (a.length !== b.length) {\n return false;\n }\n\n for (let i = 0; i < a.length; i++) {\n if (a[i] !== b[i]) { return false; }\n }\n return true;\n}\n\n// Gives a set representing the set difference a - b.\nexport function setDifference(a: Set, b: Set): Set {\n const c = new Set();\n for (const ai of a) {\n if (!b.has(ai)) { c.add(ai); }\n }\n return c;\n}\n\n// Like array.indexOf, but works with array-like objects like HTMLCollection.\nexport function indexOf(arrayLike: ArrayLike, item: T): number {\n return Array.prototype.indexOf.call(arrayLike, item);\n}\n\n/**\n * Removes a value from the given array. Only the first instance is removed.\n * Returns true on success, false if the value was not found.\n */\nexport function arrayRemove(array: T[], value: T): boolean {\n const index = array.indexOf(value);\n if (index === -1) {\n return false;\n }\n array.splice(index, 1);\n return true;\n}\n\n/**\n * Inserts value into the array before nextValue, or at the end if nextValue is not found.\n */\nexport function arrayInsertBefore(array: T[], value: T, nextValue: T): void {\n const index = array.indexOf(nextValue);\n if (index === -1) {\n array.push(value);\n } else {\n array.splice(index, 0, value);\n }\n}\n\n/**\n * Extends the first array with the second. Like native push, but adds all values in anotherArray.\n */\nexport function arrayExtend(array: T[], anotherArray: T[]): void {\n for (let i = 0, len = anotherArray.length; i < len; i++) {\n array.push(anotherArray[i]);\n }\n}\n\n/**\n * Copies count items from fromArray to toArray, copying in a forward direction (which matters\n * when the arrays are the same and source and destination indices overlap).\n *\n * See test/common/arraySplice.js for alternative implementations with timings, from which this\n * one is chosen as consistently among the faster ones.\n */\nexport function arrayCopyForward(toArray: T[], toStart: number,\n fromArray: ArrayLike, fromStart: number, count: number): void {\n const end = toStart + count;\n for (const xend = end - 7; toStart < xend; fromStart += 8, toStart += 8) {\n toArray[toStart] = fromArray[fromStart];\n toArray[toStart + 1] = fromArray[fromStart + 1];\n toArray[toStart + 2] = fromArray[fromStart + 2];\n toArray[toStart + 3] = fromArray[fromStart + 3];\n toArray[toStart + 4] = fromArray[fromStart + 4];\n toArray[toStart + 5] = fromArray[fromStart + 5];\n toArray[toStart + 6] = fromArray[fromStart + 6];\n toArray[toStart + 7] = fromArray[fromStart + 7];\n }\n for (; toStart < end; ++fromStart, ++toStart) {\n toArray[toStart] = fromArray[fromStart];\n }\n}\n\n/**\n * Copies count items from fromArray to toArray, copying in a backward direction (which matters\n * when the arrays are the same and source and destination indices overlap).\n *\n * See test/common/arraySplice.js for alternative implementations with timings, from which this\n * one is chosen as consistently among the faster ones.\n */\nexport function arrayCopyBackward(toArray: T[], toStart: number,\n fromArray: ArrayLike, fromStart: number, count: number): void {\n let i = toStart + count - 1, j = fromStart + count - 1;\n for (const xStart = toStart + 7; i >= xStart; i -= 8, j -= 8) {\n toArray[i] = fromArray[j];\n toArray[i - 1] = fromArray[j - 1];\n toArray[i - 2] = fromArray[j - 2];\n toArray[i - 3] = fromArray[j - 3];\n toArray[i - 4] = fromArray[j - 4];\n toArray[i - 5] = fromArray[j - 5];\n toArray[i - 6] = fromArray[j - 6];\n toArray[i - 7] = fromArray[j - 7];\n }\n for (; i >= toStart; --i, --j) {\n toArray[i] = fromArray[j];\n }\n}\n\n/**\n * Appends a slice of fromArray to the end of toArray.\n *\n * See test/common/arraySplice.js for alternative implementations with timings, from which this\n * one is chosen as consistently among the faster ones.\n */\nexport function arrayAppend(toArray: T[], fromArray: ArrayLike, fromStart: number, count: number): void {\n if (count === 1) {\n toArray.push(fromArray[fromStart]);\n } else {\n const len = toArray.length;\n toArray.length = len + count;\n arrayCopyForward(toArray, len, fromArray, fromStart, count);\n }\n}\n\n/**\n * Splices array arrToInsert into target starting at the given start index.\n * This implementation tries to be smart by avoiding allocations, appending to the array\n * contiguously, then filling in the gap.\n *\n * See test/common/arraySplice.js for alternative implementations with timings, from which this\n * one is chosen as consistently among the faster ones.\n */\nexport function arraySplice(target: T[], start: number, arrToInsert: ArrayLike): T[] {\n const origLen = target.length;\n const tailLen = origLen - start;\n const insLen = arrToInsert.length;\n target.length = origLen + insLen;\n if (insLen > tailLen) {\n arrayCopyForward(target, origLen, arrToInsert, tailLen, insLen - tailLen);\n arrayCopyForward(target, start + insLen, target, start, tailLen);\n arrayCopyForward(target, start, arrToInsert, 0, tailLen);\n } else {\n arrayCopyForward(target, origLen, target, origLen - insLen, insLen);\n arrayCopyBackward(target, start + insLen, target, start, tailLen - insLen);\n arrayCopyForward(target, start, arrToInsert, 0, insLen);\n }\n return target;\n}\n\n// Type for a compare func that returns a positive, negative, or zero value, as used for sorting.\nexport type CompareFunc = (a: T, b: T) => number;\n\n/**\n * Returns the index at which the given element can be inserted to keep the array sorted.\n * This is equivalent to underscore's sortedIndex and python's bisect_left.\n * @param {Array} array - sorted array of elements based on the given compareFunc\n * @param {object} elem - object to be inserted in the given array\n * @param {function} compareFunc - compares 2 elements. Returns a pos value if the 1st element is\n * larger, 0 if they're equal, a neg value if the 2nd is larger.\n */\nexport function sortedIndex(array: ArrayLike, elem: T, compareFunc: CompareFunc): number {\n let lo = 0, mid;\n let hi = array.length;\n\n if (array.length === 0) { return 0; }\n while (lo < hi) {\n mid = Math.floor((lo + hi) / 2);\n if (compareFunc(array[mid], elem) < 0) { // mid < elem\n lo = mid + 1;\n } else {\n hi = mid;\n }\n }\n return lo;\n}\n\n/**\n * Returns true if an array contains duplicate values.\n * Values are considered equal if their toString() representations are equal.\n */\nexport function hasDuplicates(array: any[]): boolean {\n const prevVals = Object.create(null);\n for (const value of array) {\n if (value in prevVals) {\n return true;\n }\n prevVals[value] = true;\n }\n return false;\n}\n\n/**\n * Counts the number of items in array which satisfy the callback.\n */\nexport function countIf(array: readonly T[], callback: (item: T) => boolean): number {\n let count = 0;\n array.forEach((item) => {\n if (callback(item)) { count++; }\n });\n return count;\n}\n\n/**\n * For two parallel arrays, calls mapFunc(a[i], b[i]) for each pair of corresponding elements, and\n * returns an array of the results.\n */\nexport function map2(array1: ArrayLike, array2: ArrayLike, mapFunc: (a: T, b: U) => V): V[] {\n const len = array1.length;\n const result: V[] = new Array(len);\n for (let i = 0; i < len; i++) {\n result[i] = mapFunc(array1[i], array2[i]);\n }\n return result;\n}\n\n/**\n * Takes a 2d array returns a new matrix with r rows and c columns\n * @param [Array] dataMatrix: a 2d array\n * @param [Number] r: final row length\n * @param [Number] c: final column length\n */\nexport function growMatrix(dataMatrix: T[][], r: number, c: number): T[][] {\n const colArr = dataMatrix.map(colVals =>\n Array.from({ length: c }, (_v, k) => colVals[k % colVals.length]),\n );\n return Array.from({ length: r }, (_v, k) => colArr[k % colArr.length]);\n}\n\n/**\n * Returns a function that compares two elements based on multiple sort keys and the\n * given compare functions.\n * Elements are compared using the sort key functions with index 0 having the greatest priority.\n * Subsequent sort key functions are used as tie breakers.\n * @param {function Array} sortKeyFuncs - a list of sort key functions.\n * @param {function Array} compareKeyFuncs - a list of comparison functions parallel to sortKeyFuncs\n * Each compare function must satisfy the comparison invariant:\n * If compare(a, b) > 0 then a > b,\n * If compare(a, b) < 0 then a < b,\n * If compare(a, b) == 0 then a == b,\n * @param {Array of 1/-1's} optAscending - Comparison on sortKeyFuncs[i] is inverted if optAscending[i] == -1\n */\nexport function multiCompareFunc(sortKeyFuncs: readonly ((a: T) => U)[],\n compareFuncs: ArrayLike>,\n optAscending?: number[]): CompareFunc {\n if (sortKeyFuncs.length !== compareFuncs.length) {\n throw new Error(\"Number of sort key funcs must be the same as the number of compare funcs\");\n }\n const ascending = optAscending || sortKeyFuncs.map(() => 1);\n return function(a: T, b: T): number {\n let compareOutcome, keyA, keyB;\n for (let i = 0; i < compareFuncs.length; i++) {\n keyA = sortKeyFuncs[i](a);\n keyB = sortKeyFuncs[i](b);\n compareOutcome = compareFuncs[i](keyA, keyB);\n if (compareOutcome !== 0) { return ascending[i] * compareOutcome; }\n }\n return 0;\n };\n}\n\nexport function nativeCompare(a: T, b: T): number {\n return (a < b ? -1 : (a > b ? 1 : 0));\n}\n\n/**\n * Creates a function that compares objects by a property value.\n */\nexport function propertyCompare(property: keyof T) {\n return function(a: T, b: T) {\n return nativeCompare(a[property], b[property]);\n };\n}\n\n// TODO: In the future, locale should be a value associated with the document or the user.\nexport const defaultLocale = \"en-US\";\nexport const defaultCollator = new Intl.Collator(defaultLocale);\nexport const localeCompare = defaultCollator.compare;\n\n/**\n * A copy of python`s `setdefault` function.\n * Sets key in mapInst to value, if key is not already set.\n * @param {Map} mapInst: Instance of Map.\n * @param {Object} key: Key into the map.\n * @param {Object} value: Value to insert, possibly.\n */\nexport function setDefault(mapInst: Map, key: K, val: V): V {\n if (!mapInst.has(key)) { mapInst.set(key, val); }\n return mapInst.get(key)!;\n}\n\n/**\n * Similar to Python's `setdefault`: returns the key `key` from `mapInst`, or if it's not there, sets\n * it to the result buildValue().\n */\nexport function getSetMapValue(mapInst: Map, key: K, buildValue: () => V): V {\n if (!mapInst.has(key)) { mapInst.set(key, buildValue()); }\n return mapInst.get(key)!;\n}\n\n/**\n * If key is in mapInst, remove it and return its value, else return `undefined`.\n * @param {Map} mapInst: Instance of Map.\n * @param {Object} key: Key into the map to remove.\n */\nexport function popFromMap(mapInst: Map, key: K): V | undefined {\n const value = mapInst.get(key);\n mapInst.delete(key);\n return value;\n}\n\n/**\n * For each encountered value in `values`, increment the corresponding counter in `valueCounts`.\n */\nexport function addCountsToMap(valueCounts: Map, values: Iterable,\n mapFunc: (v: any) => any = identity) {\n for (const v of values) {\n const mappedValue = mapFunc(v);\n valueCounts.set(mappedValue, (valueCounts.get(mappedValue) || 0) + 1);\n }\n}\n\n/**\n * Returns whether one Set is a subset of another.\n */\nexport function isSubset(smaller: Set, larger: Set): boolean {\n for (const value of smaller) {\n if (!larger.has(value)) {\n return false;\n }\n }\n return true;\n}\n\n/**\n * Merges the contents of two or more objects together into the first object, recursing into\n * nested objects and arrays (like jquery.extend(true, ...)).\n * @param {Object} target - The object to modify. Use {} to create a new merged object.\n * @param {Object} ... - Additional objects from which to copy properties into target.\n * @returns {Object} The first argument, target, modified.\n */\nexport function deepExtend(target: any, _varArgObjects: any): any {\n for (let i = 1; i < arguments.length; i++) {\n const object = arguments[i];\n // Extend the base object\n for (const name in object) {\n if (!object.hasOwnProperty(name)) { continue; }\n let src = object[name];\n if (src === target || src === undefined) {\n // Prevent one kind of infinite loop, as JQuery's extend does, and skip undefined values.\n continue;\n }\n\n if (src) {\n // Recurse if we're merging plain objects or arrays\n const tgt = target[name];\n if (Array.isArray(src)) {\n src = deepExtend(tgt && Array.isArray(tgt) ? tgt : [], src);\n } else if (typeof src === \"object\") {\n src = deepExtend(tgt && typeof tgt === \"object\" ? tgt : {}, src);\n }\n }\n target[name] = src;\n }\n }\n // Return the modified object\n return target;\n}\n\n/**\n * Returns a human-readable string containing a number of bytes, KB, or MB.\n * @param {Number} bytes. Number of bytes.\n * @returns {String} A description such as \"4.1KB\".\n */\nexport function byteString(bytes: number): string {\n if (bytes < 1024) {\n return bytes + \"B\";\n } else if (bytes < 1024 * 1024) {\n return (bytes / 1024).toFixed(1) + \"KB\";\n } else {\n return (bytes / 1024 / 1024).toFixed(1) + \"MB\";\n }\n}\n\n/**\n * Creates a new object mapping each key in keysArray to the value returned by callback.\n * @param {Array} keysArray - Array of strings to use as the properties of the returned object.\n * @param {Function} callback - Function that produces the value for each key. Called in the same\n * way as array.map() calls its callbacks.\n * @param {Object} optThisArg - Value to use as `this` when executing callback.\n * @returns {Object} - object mapping keys from `keysArray` to values returned by `callback`.\n */\nexport function mapToObject(keysArray: string[], callback: (key: string) => T,\n optThisArg: any): { [key: string]: T } {\n const values: T[] = keysArray.map(callback, optThisArg);\n const map: { [key: string]: T } = {};\n for (let i = 0; i < keysArray.length; i++) {\n map[keysArray[i]] = values[i];\n }\n return map;\n}\n\n/**\n * Remove the specified elements from the array, with the elements specified by\n * their index. The array arr is modified in-place. The indexes must be provided\n * in order, sorted lowest to highest, with no duplicates, or out-of-bound indices,\n * etc (this method does no error checking; it is used in place of lodash-pullAt\n * for performance reasons).\n */\nexport function pruneArray(arr: T[], indexes: number[]) {\n if (indexes.length === 0) { return; }\n if (indexes.length === 1) {\n arr.splice(indexes[0], 1);\n return;\n }\n const len = arr.length;\n let arrAt = 0;\n let indexesAt = 0;\n for (let i = 0; i < len; i++) {\n if (i === indexes[indexesAt]) {\n indexesAt++;\n continue;\n }\n if (i !== arrAt) {\n arr[arrAt] = arr[i];\n }\n arrAt++;\n }\n arr.length = arrAt;\n}\n\n/**\n * A List of python identifiers; the result of running keywords.kwlist in Python 3.9,\n * plus additional illegal identifiers None, False, True\n * Using [] instead of new Array causes a \"comprehension error\" for some reason\n */\nconst _kwlist = [\"False\", \"None\", \"True\", \"and\", \"as\", \"assert\", \"async\", \"await\", \"break\",\n \"class\", \"continue\", \"def\", \"del\", \"elif\", \"else\", \"except\", \"finally\",\n \"for\", \"from\", \"global\", \"if\", \"import\", \"in\", \"is\", \"lambda\", \"nonlocal\",\n \"not\", \"or\", \"pass\", \"raise\", \"return\", \"try\", \"while\", \"with\", \"yield\"];\n/**\n * Given an arbitrary string, makes substitutions to make it a valid SQL/Python identifier.\n * Corresponds to sandbox/grist/gencode.sanitize_ident\n */\nexport function sanitizeIdent(ident: string, prefix?: string) {\n prefix = prefix || \"c\";\n // Remove non-alphanumeric non-_ chars\n ident = ident.replace(/[^a-zA-Z0-9_]+/g, \"_\");\n // Remove leading and trailing _\n ident = ident.replace(/^_+|_+$/g, \"\");\n // Place prefix at front if the beginning isn't a number\n ident = ident.replace(/^(?=[0-9])/g, prefix);\n // Append prefix until it is not python keyword\n while (_kwlist.includes(ident)) {\n ident = prefix + ident;\n }\n return ident;\n}\n\n/**\n * Clone a function, returning a function object that represents a brand new function with the\n * same code. If the same function is used with different argument types, it would prevent JS V8\n * engine optimizations (or cause it to deoptimize it). If different clones are called with\n * different argument types, they can be optimized independently.\n *\n * As with all micro-optimizations, only do this when the optimization matters.\n */\nexport function cloneFunc(fn: Function): Function { /* jshint evil:true */ // suppress eval warning.\n return eval(\"(\" + fn.toString() + \")\");\n}\n\n/**\n * Generates a random id using a sequence of uppercase alphanumeric characters\n * preceded by an optional prefix.\n */\nexport function genRandomId(len: number, optPrefix?: string): string {\n const chars = \"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ\";\n let ret = optPrefix || \"\";\n for (let i = 0; i < len; i++) {\n ret += chars[Math.floor(Math.random() * chars.length)];\n }\n return ret;\n}\n\n/**\n * Scans through two sorted arrays, calling a function on each item or pair of items\n * for every present key in order.\n * @param {Array} arrA - First array to scan. NOTE: Should be sorted by the key value.\n * @param {Array} arrB - Second array to scan. NOTE: Should be sorted by the key value.\n * @param {Function} callback - Called with an item from arrA as the first argument and an\n * item from arrB as the second. Called for every unique key in order, either with one of the\n * arguments null if the key is present only in one array, or both non-null if the key is\n * present in both arrays. NOTE: Key values should not be null.\n * @param {Function} optKeyFunc - Optional function to map each array value to a sort key.\n * Defaults to the identity function.\n */\nexport function sortedScan(arrA: ArrayLike, arrB: ArrayLike,\n callback: (a: T | null, B: U | null) => void,\n optKeyFunc?: (item: T | U) => any) {\n const keyFunc = optKeyFunc || identity;\n let i = 0, j = 0;\n while (i < arrA.length || j < arrB.length) {\n const a = arrA[i], b = arrB[j];\n const keyA = i < arrA.length ? keyFunc(a) : null;\n const keyB = j < arrB.length ? keyFunc(b) : null;\n if (keyA !== null && (keyB === null || keyA < keyB)) {\n callback(a, null);\n i++;\n } else if (keyA === null || keyA > keyB) {\n callback(null, b);\n j++;\n } else {\n callback(a, b);\n i++;\n j++;\n }\n }\n}\n\n/**\n * Returns the time in ms to wait until attempting another connection.\n * @param {Number} attemptNumber - Reconnect attempt number starting at 0.\n * @param {Array} intervals - Array of reconnect intervals in ms.\n * @returns {Number}\n */\nexport function getReconnectTimeout(attemptNumber: number, intervals: ArrayLike): number {\n if (attemptNumber >= intervals.length) {\n // Add an additional wait time if already at max attempts.\n const timeout = intervals[intervals.length - 1];\n return timeout + Math.random() * timeout;\n } else {\n return intervals[attemptNumber];\n }\n}\n\n/**\n * Returns whether the given email is a valid formatted email string.\n * @param {String} email - Email to test.\n * @returns {Boolean}\n */\nexport function isEmail(email: string): boolean {\n return EMAIL_RE.test(email.toLowerCase());\n}\n\n/*\n * Takes an observable and returns a promise for when the observable's value matches the given\n * predicate. It then unsubscribes from the observable, and returns its value.\n * If a predicate is not given, resolves to the observable values as soon as it's truthy.\n */\nexport function waitObs(observable: KoObservable, predicate: (value: T) => boolean = Boolean): Promise {\n return new Promise((resolve, _reject) => {\n const value = observable.peek();\n if (predicate(value)) { return resolve(value); }\n const sub = observable.subscribe((val: T) => {\n if (predicate(val)) {\n sub.dispose();\n resolve(val);\n }\n });\n });\n}\n\n/**\n * Same as waitObs but for grainjs observables.\n */\nexport async function waitGrainObs(observable: Observable): Promise>;\nexport async function waitGrainObs(observable: Observable, predicate?: (value: T) => boolean): Promise;\nexport async function waitGrainObs(observable: Observable,\n predicate: (value: T) => boolean = Boolean): Promise {\n let sub: Listener | undefined;\n const res: T = await new Promise((resolve, _reject) => {\n const value = observable.get();\n if (predicate(value)) { return resolve(value); }\n sub = observable.addListener((val: T) => {\n if (predicate(val)) {\n resolve(val);\n }\n });\n });\n if (sub) { sub.dispose(); }\n return res;\n}\n\n// `dom.style` does not work here because custom css property (ie: `--foo`) needs to be set using\n// `style.setProperty` (credit: https://vanseodesign.com/css/custom-properties-and-javascript/).\n// TODO: consider making PR to fix `dom.style` in grainjs.\nexport function inlineStyle(property: string, valueObs: BindableValue): DomElementMethod {\n return elem => subscribeElem(elem, valueObs, (val) => {\n elem.style.setProperty(property, String(val ?? \"\"));\n });\n}\n\n/**\n * Class to maintain a chain of promise-returning callbacks. All scheduled callbacks will be\n * called in order as long as the previous one is successful. If a callback fails is rejected,\n * already-scheduled callbacks will be skipped, but newly-scheduled ones will be run.\n */\nexport class PromiseChain {\n private _last: Promise = Promise.resolve();\n\n // Adds a callback to the chain. If the callback runs, the return value is the return value of\n // the callback. If it's skipped due to a failure earlier in the chain, the return value is the\n // rejection with the message \"Skipped due to an earlier error\".\n public add(nextCB: () => Promise): Promise {\n const next = this._last.catch(() => { throw new Error(\"Skipped due to an earlier error\"); }).then(nextCB);\n // If any callback fails, all queued ones will be skipped. Here we reset the chain, so that\n // callbacks added later do get run.\n next.catch(() => { this._last = Promise.resolve(); });\n this._last = next;\n return next;\n }\n}\n\n/**\n * Indicates if a hex color value, e.g. '#000000', is darker than the given value.\n * Darkness is measured from 0..255, where 0 is the darkest and 255 is the lightest.\n *\n * Taken from: https://stackoverflow.com/questions/12043187/how-to-check-if-hex-color-is-too-black\n */\nexport function isColorDark(hexColor: string, isDarkBelow: number = 220): boolean {\n const c = hexColor.substring(1); // strip #\n const rgb = parseInt(c, 16); // convert rrggbb to decimal\n // Extract RGB components\n const r = (rgb >> 16) & 0xff; const g = (rgb >> 8) & 0xff; const b = (rgb >> 0) & 0xff;\n const luma = 0.2126 * r + 0.7152 * g + 0.0722 * b; // per ITU-R BT.709\n return luma < isDarkBelow;\n}\n\n/**\n * Returns true if val is a valid hex color value. For instance: #aabbaa is valid, #aabba is not. Do\n * not accept neither short notation nor hex with transparency, ie: #aab, #aabb and #aabbaabb are\n * invalid.\n */\nexport function isValidHex(val: unknown): val is string {\n if (typeof val !== \"string\") {\n return false;\n }\n\n return /^#([0-9A-F]{6})$/i.test(val);\n}\n\n/**\n * Resolves to true if promise is still pending after msec milliseconds have passed. Otherwise\n * returns false, including when promise is rejected.\n */\nexport async function timeoutReached(\n msec: number, promise: Promise, options: { rethrow: boolean } = { rethrow: false },\n): Promise {\n // For test purposes, support negative timeout, by failing\n // immediately.\n if (msec < 0) {\n return true;\n }\n const timedOut = {};\n // Be careful to clean up the timer after ourselves, so it doesn't remain in the event loop.\n let timer: NodeJS.Timeout;\n const delayPromise = new Promise((resolve) => { timer = setTimeout(() => resolve(timedOut), msec); });\n try {\n const res = await Promise.race([promise, delayPromise]);\n return res == timedOut;\n } catch (err) {\n if (options.rethrow) {\n throw err;\n }\n return false;\n } finally {\n clearTimeout(timer!);\n }\n}\n\n/**\n * Returns a promise that resolves to true if promise takes longer than timeoutMsec to resolve. If not\n * or if promise throws returns false. Same as timeoutReached(), with reversed order of arguments.\n */\nexport async function isLongerThan(promise: Promise, timeoutMsec: number): Promise {\n return timeoutReached(timeoutMsec, promise);\n}\n\n/**\n * Returns true if the parameter, when rendered as a string, matches\n * 1, on, or true (case insensitively). Useful for processing query\n * parameters that may have been manually set.\n */\nexport function isAffirmative(parameter: any): boolean {\n return [\"1\", \"on\", \"true\", \"yes\"].includes(String(parameter).toLowerCase());\n}\n\n/**\n * Returns whether a value is neither null nor undefined, with a type guard for the return type.\n *\n * This is particularly useful for filtering, e.g. if `array` includes values of type\n * T|null|undefined, then TypeScript can tell that `array.filter(isNonNullish)` has the type T[].\n */\nexport function isNonNullish(value: T | null | undefined): value is T {\n return value !== null && value !== undefined;\n}\n\n/**\n * Ensures that a value is truthy, with a type guard for the return type.\n */\nexport function truthy(value: T | null | undefined): value is Exclude {\n return Boolean(value);\n}\n\n/**\n * Returns the value of both grainjs and knockout observable without creating a dependency.\n */\nexport const unwrap: UseCB = (obs: ISubscribable) => {\n if (\"_getDepItem\" in obs) {\n return obs.get();\n }\n return (obs as ko.Observable).peek();\n};\n\n/**\n * Subscribes to BindableValue\n */\nexport function useBindable(use: UseCBOwner, obs: BindableValue): T {\n if (obs === null || obs === undefined) { return obs; }\n\n const smth = obs as any;\n\n // If knockout\n if (typeof smth === \"function\" && \"peek\" in smth) { return use(smth) as T; }\n // If grainjs Observable or Computed\n if (typeof smth === \"object\" && \"_getDepItem\" in smth) { return use(smth) as T; }\n // If use function ComputedCallback\n if (typeof smth === \"function\") { return smth(use) as T; }\n\n return obs as T;\n}\n\n/**\n * Useful helper for simple boolean negation.\n */\nexport const not = (\n obs: Observable | IKnockoutReadObservable | boolean | undefined | null,\n) => (use: UseCBOwner) => {\n if (typeof obs === \"boolean\") { return !obs; }\n if (obs === null || obs === undefined) { return true; }\n return !use(obs);\n};\n\n/**\n * Get a set of up to `count` distinct values of `values`.\n */\nexport function getDistinctValues(values: readonly T[], count: number = Infinity): Set {\n const distinct = new Set();\n // Add values to the set until it reaches the desired size, or until there are no more values.\n for (let i = 0; i < values.length && distinct.size < count; i++) {\n distinct.add(values[i]);\n }\n return distinct;\n}\n\n/**\n * Asserts that variable `name` has a non-nullish `value`.\n */\nexport function assertIsDefined(name: string, value: T): asserts value is NonNullable {\n if (value === undefined || value === null) {\n throw new Error(`Expected '${name}' to be defined, but received ${value}`);\n }\n}\n\n/**\n * Calls function `fn`, passes any thrown errors to function `recover`, and finally calls `fn`\n * once more if `recover` doesn't throw.\n */\nexport async function retryOnce(fn: () => Promise, recover: (e: unknown) => Promise): Promise {\n try {\n return await fn();\n } catch (e) {\n await recover(e);\n return await fn();\n }\n}\n\n/**\n * Checks if value is 'empty' (like null, undefined, empty string, empty array/set/map, empty object).\n * Values like 0, true, false are not empty.\n */\nexport function notSet(value: any) {\n return value === undefined || value === null || value === \"\" ||\n (Array.isArray(value) && !value.length) ||\n (typeof value === \"object\" && !Object.keys(value).length) ||\n ([\"[object Map]\", \"[object Set\"].includes(value.toString()) && !value.size);\n}\n\n/**\n * Checks if value is 'empty', if it is, returns the default value (which is null).\n */\nexport function ifNotSet(value: any, def: any = null) {\n return notSet(value) ? def : value;\n}\n\n/**\n * Creates a computed observable with a nested owner that can be used to dispose,\n * any disposables created inside the computed. Similar to domComputedOwned method.\n */\nexport function computedOwned(\n owner: IDisposableOwner,\n func: (owner: IDisposableOwner, use: UseCBOwner) => T,\n): Computed {\n const holder = Holder.create(owner);\n return Computed.create(owner, (use) => {\n const computedOwner = MultiHolder.create(holder);\n return func(computedOwner, use);\n });\n}\n\nexport type Constructor = new (...args: any[]) => T;\n\n/**\n * Simple memoization function that caches the result of a function call based on its arguments.\n * Unlike lodash's memoize, it uses all arguments to generate the key.\n */\nexport function cached(fn: T): T {\n const dict = new Map();\n const impl = (...args: any[]) => {\n const key = JSON.stringify(args);\n if (!dict.has(key)) {\n dict.set(key, (fn as any)(...args));\n }\n return dict.get(key);\n };\n return impl as any as T;\n}\n\n/**\n * Converts a duration string like \"1d\", \"2h\", \"14m\", \"10s\" to seconds.\n */\nexport function inSeconds(text: string): number {\n const match = text.match(/^(\\d+)([smhd])$/);\n if (!match) {\n throw new Error(`Invalid duration: ${text}`);\n }\n const [, value, unit] = match;\n const seconds = parseInt(value, 10);\n switch (unit) {\n case \"s\": return seconds;\n case \"m\": return seconds * 60;\n case \"h\": return seconds * 60 * 60;\n case \"d\": return seconds * 60 * 60 * 24;\n default: throw new Error(`Invalid duration unit: ${unit}`);\n }\n}\n" }, { "path": "app/common/isHiddenTable.ts", "content": "import { TableData } from \"app/common/TableData\";\nimport { UIRowId } from \"app/plugin/GristAPI\";\n\n/**\n * Return whether a table (identified by the rowId of its metadata record) should\n * normally be hidden from the user (e.g. as an option in the page-widget picker).\n */\nexport function isHiddenTable(tablesData: TableData, tableRef: UIRowId): boolean {\n const tableId = tablesData.getValue(tableRef, \"tableId\") as string | undefined;\n // The `!tableId` check covers the case of censored tables (see isTableCensored() below).\n return !tableId || isSummaryTable(tablesData, tableRef) || tableId.startsWith(\"GristHidden_\");\n}\n\n/**\n * Return whether a table (identified by the rowId of its metadata record) is a\n * summary table.\n */\nexport function isSummaryTable(tablesData: TableData, tableRef: UIRowId): boolean {\n return tablesData.getValue(tableRef, \"summarySourceTable\") !== 0;\n}\n\n// Check if a table record (from _grist_Tables) is censored.\n// Metadata records get censored by clearing certain of their fields, so it's expected that a\n// record may exist even though various code should consider it as hidden.\nexport function isTableCensored(tablesData: TableData, tableRef: UIRowId): boolean {\n const tableId = tablesData.getValue(tableRef, \"tableId\");\n return !tableId;\n}\n\n/**\n * Returns whether this is a metadata table, as opposed to a user table.\n */\nexport function isMetadataTable(tableId: string): boolean {\n return tableId.startsWith(\"_grist\");\n}\n" }, { "path": "app/common/loginProviders.ts", "content": "// Special provider that uses default user, which effectively means no authentication.\nexport const MINIMAL_PROVIDER_KEY = \"minimal\";\n\n// OpenID Connect provider key.\nexport const OIDC_PROVIDER_KEY = \"oidc\";\n\n// ForwardAuth provider key.\nexport const FORWARD_AUTH_PROVIDER_KEY = \"forward-auth\";\n\n// This provider is only available in grist-ee version.\nexport const GRIST_CONNECT_PROVIDER_KEY = \"grist-connect\";\n\n// getgrist.com provider key.\nexport const GETGRIST_COM_PROVIDER_KEY = \"getgrist.com\";\n\n// SAML provider key.\nexport const SAML_PROVIDER_KEY = \"saml\";\n\n// Deprecated/unmaintained providers, hidden unless already configured or active.\nexport const DEPRECATED_PROVIDERS: string[] = [\n GRIST_CONNECT_PROVIDER_KEY,\n];\n" }, { "path": "app/common/marshal.ts", "content": "/**\n * Module for serializing data in the format of Python 'marshal' module. It's used for\n * communicating with the Python-based formula engine running in a Pypy sandbox. It supports\n * version 0 of python marshalling format, which is what the Pypy sandbox supports.\n *\n * Usage:\n * Marshalling:\n * const marshaller = new Marshaller({version: 2});\n * marshaller.marshal(value);\n * marshaller.marshal(value);\n * const buf = marshaller.dump(); // Leaves the marshaller empty.\n *\n * Unmarshalling:\n * const unmarshaller = new Unmarshaller();\n * unmarshaller.on('value', function(value) { ... });\n * unmarshaller.push(buffer);\n * unmarshaller.push(buffer);\n *\n * In Python, and in the marshalled format, there is a distinction between strings and unicode\n * objects. In JS, there is a good correspondence to Uint8Array objects and strings, respectively.\n * Python unicode objects always become JS strings. JS Uint8Arrays always become Python strings.\n *\n * JS strings become Python unicode objects, but can be marshalled to Python strings with\n * 'stringToBuffer' option. Similarly, Python strings become JS Uint8Arrays, but can be\n * unmarshalled to JS strings if 'bufferToString' option is set.\n */\nimport { BigInt } from \"app/common/BigInt\";\nimport MemBuffer from \"app/common/MemBuffer\";\n\nimport { EventEmitter } from \"events\";\nimport * as util from \"util\";\n\nexport interface MarshalOptions {\n stringToBuffer?: boolean;\n version?: number;\n\n // True if we want keys in dicts to be buffers.\n // It is convenient to have some freedom here to simplify implementation\n // of marshaling for some SQLite wrappers. This flag was initially\n // introduced for a fork of Grist using better-sqlite3, and I don't\n // remember exactly what the issues were.\n keysAreBuffers?: boolean;\n}\n\nexport interface UnmarshalOptions {\n bufferToString?: boolean;\n}\n\nfunction ord(str: string): number {\n return str.charCodeAt(0);\n}\n\n/**\n * Type codes used for python marshalling of values.\n * See pypy: rpython/translator/sandbox/_marshal.py.\n */\nconst marshalCodes = {\n NULL: ord(\"0\"),\n NONE: ord(\"N\"),\n FALSE: ord(\"F\"),\n TRUE: ord(\"T\"),\n STOPITER: ord(\"S\"),\n ELLIPSIS: ord(\".\"),\n INT: ord(\"i\"),\n INT64: ord(\"I\"),\n /*\n BFLOAT, for 'binary float', is an encoding of float that just encodes the bytes of the\n double in standard IEEE 754 float64 format. It is used by Version 2+ of Python's marshal\n module. Previously (in versions 0 and 1), the FLOAT encoding is used, which stores floats\n through their string representations.\n\n Version 0 (FLOAT) is mandatory for system calls within the sandbox, while Version 2 (BFLOAT)\n is recommended for Grist's communication because it is more efficient and faster to\n encode/decode\n */\n BFLOAT: ord(\"g\"),\n FLOAT: ord(\"f\"),\n COMPLEX: ord(\"x\"),\n LONG: ord(\"l\"),\n STRING: ord(\"s\"),\n INTERNED: ord(\"t\"),\n STRINGREF: ord(\"R\"),\n TUPLE: ord(\"(\"),\n LIST: ord(\"[\"),\n DICT: ord(\"{\"),\n CODE: ord(\"c\"),\n UNICODE: ord(\"u\"),\n UNKNOWN: ord(\"?\"),\n SET: ord(\"<\"),\n FROZENSET: ord(\">\"),\n};\n\ntype MarshalCode = keyof typeof marshalCodes;\n\n// A little hack to test if the value is a 32-bit integer. Actually, for Python, int might be up\n// to 64 bits (if that's the native size), but this is simpler.\n// See http://stackoverflow.com/questions/3885817/how-to-check-if-a-number-is-float-or-integer.\nfunction isInteger(n: number): boolean {\n // Float have +0.0 and -0.0. To represent -0.0 precisely, we have to use a float, not an int\n // (see also https://stackoverflow.com/questions/7223359/are-0-and-0-the-same).\n return n === +n && n === (n | 0) && !Object.is(n, -0.0);\n}\n\n// ----------------------------------------------------------------------\n\n/**\n * To force a value to be serialized using a particular representation (e.g. a number as INT64),\n * wrap it into marshal.wrap('INT64', value) and serialize that.\n */\nexport function wrap(codeStr: MarshalCode, value: unknown) {\n return new WrappedObj(marshalCodes[codeStr], value);\n}\n\nexport class WrappedObj {\n constructor(public code: number, public value: unknown) {}\n\n public inspect() {\n return util.inspect(this.value);\n }\n}\n\n// ----------------------------------------------------------------------\n\n/**\n * @param {Boolean} options.stringToBuffer - If set, JS strings will become Python strings rather\n * than unicode objects (as if each JS string is wrapped into MemBuffer.stringToArray(str)).\n * This flag becomes a same-named property of Marshaller, which can be set at any time.\n * @param {Number} options.version - If version >= 2, uses binary representation for floats. The\n * default version 0 formats floats as strings.\n *\n * TODO: The default should be version 2. (0 was used historically because it was needed for\n * communication with PyPy-based sandbox.)\n */\nexport class Marshaller {\n private _memBuf: MemBuffer;\n private readonly _floatCode: number;\n private readonly _stringCode: number;\n private readonly _keysAreBuffers: boolean;\n\n constructor(options?: MarshalOptions) {\n this._memBuf = new MemBuffer(undefined);\n this._floatCode = options?.version && options.version >= 2 ? marshalCodes.BFLOAT : marshalCodes.FLOAT;\n this._stringCode = options?.stringToBuffer ? marshalCodes.STRING : marshalCodes.UNICODE;\n this._keysAreBuffers = Boolean(options?.keysAreBuffers);\n }\n\n public dump(): Uint8Array {\n // asByteArray returns a view on the underlying data, and the constructor creates a new copy.\n // For some usages, we may want to avoid making the copy.\n const bytes = new Uint8Array(this._memBuf.asByteArray());\n this._memBuf.clear();\n return bytes;\n }\n\n public dumpAsBuffer(): Buffer {\n const bytes = Buffer.from(this._memBuf.asByteArray());\n this._memBuf.clear();\n return bytes;\n }\n\n public getCode(value: any) {\n switch (typeof value) {\n case \"number\": return isInteger(value) ? marshalCodes.INT : this._floatCode;\n case \"string\": return this._stringCode;\n case \"boolean\": return value ? marshalCodes.TRUE : marshalCodes.FALSE;\n case \"undefined\": return marshalCodes.NONE;\n case \"object\": {\n if (value instanceof WrappedObj) {\n return value.code;\n } else if (value === null) {\n return marshalCodes.NONE;\n } else if (value instanceof Uint8Array) {\n return marshalCodes.STRING;\n } else if (Buffer.isBuffer(value)) {\n return marshalCodes.STRING;\n } else if (Array.isArray(value)) {\n return marshalCodes.LIST;\n }\n return marshalCodes.DICT;\n }\n default: {\n throw new Error(\"Marshaller: Unsupported value of type \" + (typeof value));\n }\n }\n }\n\n public marshal(value: any): void {\n const code = this.getCode(value);\n if (value instanceof WrappedObj) {\n value = value.value;\n }\n this._memBuf.writeUint8(code);\n switch (code) {\n case marshalCodes.NULL: return;\n case marshalCodes.NONE: return;\n case marshalCodes.FALSE: return;\n case marshalCodes.TRUE: return;\n case marshalCodes.INT: return this._memBuf.writeInt32LE(value);\n case marshalCodes.INT64: return this._writeInt64(value);\n case marshalCodes.FLOAT: return this._writeStringFloat(value);\n case marshalCodes.BFLOAT: return this._memBuf.writeFloat64LE(value);\n case marshalCodes.STRING:\n return (value instanceof Uint8Array || Buffer.isBuffer(value) ?\n this._writeByteArray(value) :\n this._writeUtf8String(value));\n case marshalCodes.TUPLE: return this._writeList(value);\n case marshalCodes.LIST: return this._writeList(value);\n case marshalCodes.DICT: return this._writeDict(value);\n case marshalCodes.UNICODE: return this._writeUtf8String(value);\n // None of the following are supported.\n case marshalCodes.STOPITER:\n case marshalCodes.ELLIPSIS:\n case marshalCodes.COMPLEX:\n case marshalCodes.LONG:\n case marshalCodes.INTERNED:\n case marshalCodes.STRINGREF:\n case marshalCodes.CODE:\n case marshalCodes.UNKNOWN:\n case marshalCodes.SET:\n case marshalCodes.FROZENSET: throw new Error(\"Marshaller: Can't serialize code \" + code);\n default: throw new Error(\"Marshaller: Can't serialize code \" + code);\n }\n }\n\n private _writeInt64(value: number) {\n if (!isInteger(value)) {\n // TODO We could actually support 53 bits or so.\n throw new Error(\"Marshaller: int64 still only supports 32-bit ints for now: \" + value);\n }\n this._memBuf.writeInt32LE(value);\n this._memBuf.writeInt32LE(value >= 0 ? 0 : -1);\n }\n\n private _writeStringFloat(value: number) {\n // This could be optimized a bit, but it's only used in V0 marshalling, which is only used in\n // sandbox system calls, which don't really ever use floats anyway.\n const bytes = MemBuffer.stringToArray(value.toString());\n if (bytes.byteLength >= 127) {\n throw new Error(\"Marshaller: Trying to write a float that takes \" + bytes.byteLength + \" bytes\");\n }\n this._memBuf.writeUint8(bytes.byteLength);\n this._memBuf.writeByteArray(bytes);\n }\n\n private _writeByteArray(value: Uint8Array | Buffer) {\n // This works for both Uint8Arrays and Node Buffers.\n this._memBuf.writeInt32LE(value.length);\n this._memBuf.writeByteArray(value);\n }\n\n private _writeUtf8String(value: string) {\n const offset = this._memBuf.size();\n // We don't know the length until we write the value.\n this._memBuf.writeInt32LE(0);\n this._memBuf.writeString(value);\n const byteLength = this._memBuf.size() - offset - 4;\n // Overwrite the 0 length we wrote earlier with the correct byte length.\n this._memBuf.asDataView.setInt32(this._memBuf.startPos + offset, byteLength, true);\n }\n\n private _writeList(array: unknown[]) {\n this._memBuf.writeInt32LE(array.length);\n for (const item of array) {\n this.marshal(item);\n }\n }\n\n private _writeDict(obj: { [key: string]: any }) {\n const keys = Object.keys(obj);\n keys.sort();\n for (const key of keys) {\n this.marshal(this._keysAreBuffers ? Buffer.from(key) : key);\n this.marshal(obj[key]);\n }\n this._memBuf.writeUint8(marshalCodes.NULL);\n }\n}\n\n// ----------------------------------------------------------------------\n\nconst TwoTo32 = 0x100000000; // 2**32\nconst TwoTo15 = 0x8000; // 2**15\n\n/**\n * @param {Boolean} options.bufferToString - If set, Python strings will become JS strings rather\n * than Buffers (as if each decoded buffer is wrapped into `buf.toString()`).\n * This flag becomes a same-named property of Unmarshaller, which can be set at any time.\n * Note that options.version isn't needed, since this will decode both formats.\n * TODO: Integers (such as int64 and longs) that are too large for JS are currently represented as\n * decimal strings. They may need a better representation, or a configurable option.\n */\nexport class Unmarshaller extends EventEmitter {\n public memBuf: MemBuffer;\n private _consumer: any = null;\n private _lastCode: number | null = null;\n private readonly _bufferToString: boolean;\n private _emitter: (v: any) => boolean;\n private _stringTable: (string | Uint8Array)[] = [];\n\n constructor(options?: UnmarshalOptions) {\n super();\n this.memBuf = new MemBuffer(undefined);\n this._bufferToString = Boolean(options?.bufferToString);\n this._emitter = this.emit.bind(this, \"value\");\n }\n\n /**\n * Adds more data for parsing. Parsed values will be emitted as 'value' events.\n * @param {Uint8Array|Buffer} byteArray: Uint8Array or Node Buffer with bytes to parse.\n */\n public push(byteArray: Uint8Array | Buffer) {\n this.parse(byteArray, this._emitter);\n }\n\n /**\n * Adds data to parse, and calls valueCB(value) for each value parsed. If valueCB returns the\n * Boolean false, stops parsing and returns.\n */\n public parse(byteArray: Uint8Array | Buffer, valueCB: (val: any) => boolean | void) {\n this.memBuf.writeByteArray(byteArray);\n try {\n while (this.memBuf.size() > 0) {\n this._consumer = this.memBuf.makeConsumer();\n\n // Have to reset stringTable for interned strings before each top-level parse call.\n this._stringTable.length = 0;\n\n const value = this._parse();\n this.memBuf.consume(this._consumer);\n if (valueCB(value) === false) {\n return;\n }\n }\n } catch (err) {\n // If the error is `needMoreData`, we silently return. We'll retry by reparsing the message\n // from scratch after the next push(). If buffers contain complete serialized messages, the\n // cost should be minor. But this design might get very inefficient if we have big messages\n // of arrays or dictionaries.\n if (err.needMoreData) {\n if (!err.consumedData || err.consumedData > 1024) {\n console.log(\"Unmarshaller: Need more data; wasted parsing of %d bytes\", err.consumedData);\n }\n } else {\n err.message = \"Unmarshaller: \" + err.message;\n throw err;\n }\n }\n }\n\n private _parse(): unknown {\n const code = this.memBuf.readUint8(this._consumer);\n this._lastCode = code;\n switch (code) {\n case marshalCodes.NULL: return null;\n case marshalCodes.NONE: return null;\n case marshalCodes.FALSE: return false;\n case marshalCodes.TRUE: return true;\n case marshalCodes.INT: return this._parseInt();\n case marshalCodes.INT64: return this._parseInt64();\n case marshalCodes.FLOAT: return this._parseStringFloat();\n case marshalCodes.BFLOAT: return this._parseBinaryFloat();\n case marshalCodes.STRING: return this._parseByteString();\n case marshalCodes.TUPLE: return this._parseList();\n case marshalCodes.LIST: return this._parseList();\n case marshalCodes.DICT: return this._parseDict();\n case marshalCodes.UNICODE: return this._parseUnicode();\n case marshalCodes.INTERNED: return this._parseInterned();\n case marshalCodes.STRINGREF: return this._parseStringRef();\n case marshalCodes.LONG: return this._parseLong();\n // None of the following are supported.\n // case marshalCodes.STOPITER:\n // case marshalCodes.ELLIPSIS:\n // case marshalCodes.COMPLEX:\n // case marshalCodes.CODE:\n // case marshalCodes.UNKNOWN:\n // case marshalCodes.SET:\n // case marshalCodes.FROZENSET:\n default:\n throw new Error(`Unmarshaller: unsupported code \"${String.fromCharCode(code)}\" (${code})`);\n }\n }\n\n private _parseInt() {\n return this.memBuf.readInt32LE(this._consumer);\n }\n\n private _parseInt64() {\n const low = this.memBuf.readInt32LE(this._consumer);\n const hi = this.memBuf.readInt32LE(this._consumer);\n if ((hi === 0 && low >= 0) || (hi === -1 && low < 0)) {\n return low;\n }\n const unsignedLow = low < 0 ? TwoTo32 + low : low;\n if (hi >= 0) {\n return new BigInt(TwoTo32, [unsignedLow, hi], 1).toNative();\n } else {\n // This part is tricky. See unittests for check of correctness.\n return new BigInt(TwoTo32, [TwoTo32 - unsignedLow, -hi - 1], -1).toNative();\n }\n }\n\n private _parseLong() {\n // The format is a 32-bit size whose sign is the sign of the result, followed by 16-bit digits\n // in base 2**15.\n const size = this.memBuf.readInt32LE(this._consumer);\n const sign = size < 0 ? -1 : 1;\n const numDigits = size < 0 ? -size : size;\n const digits = [];\n for (let i = 0; i < numDigits; i++) {\n digits.push(this.memBuf.readInt16LE(this._consumer));\n }\n return new BigInt(TwoTo15, digits, sign).toNative();\n }\n\n private _parseStringFloat() {\n const len = this.memBuf.readUint8(this._consumer);\n const buf = this.memBuf.readString(this._consumer, len);\n return parseFloat(buf);\n }\n\n private _parseBinaryFloat() {\n return this.memBuf.readFloat64LE(this._consumer);\n }\n\n private _parseByteString(): string | Uint8Array {\n const len = this.memBuf.readInt32LE(this._consumer);\n return (this._bufferToString ?\n this.memBuf.readString(this._consumer, len) :\n this.memBuf.readByteArray(this._consumer, len));\n }\n\n private _parseInterned() {\n const s = this._parseByteString();\n this._stringTable.push(s);\n return s;\n }\n\n private _parseStringRef() {\n const index = this._parseInt();\n return this._stringTable[index];\n }\n\n private _parseList() {\n const len = this.memBuf.readInt32LE(this._consumer);\n const value = [];\n for (let i = 0; i < len; i++) {\n value[i] = this._parse();\n }\n return value;\n }\n\n private _parseDict() {\n const dict: { [key: string]: any } = {};\n while (true) {\n let key = this._parse() as string | Uint8Array;\n if (key === null && this._lastCode === marshalCodes.NULL) {\n break;\n }\n const value = this._parse();\n if (key !== null) {\n if (key instanceof Uint8Array) {\n key = MemBuffer.arrayToString(key);\n }\n dict[key as string] = value;\n }\n }\n return dict;\n }\n\n private _parseUnicode() {\n const len = this.memBuf.readInt32LE(this._consumer);\n return this.memBuf.readString(this._consumer, len);\n }\n}\n\n/**\n * Similar to python's marshal.loads(). Parses the given bytes and returns the parsed value. There\n * must not be any trailing data beyond the single marshalled value.\n */\nexport function loads(byteArray: Uint8Array | Buffer, options?: UnmarshalOptions): any {\n const unmarshaller = new Unmarshaller(options);\n let parsedValue;\n unmarshaller.parse(byteArray, function(value) {\n parsedValue = value;\n return false;\n });\n if (typeof parsedValue === \"undefined\") {\n throw new Error(\"loads: input data truncated\");\n } else if (unmarshaller.memBuf.size() > 0) {\n throw new Error(\"loads: extra bytes past end of input\");\n }\n return parsedValue;\n}\n\n/**\n * Serializes arbitrary data by first marshalling then converting to a base64 string.\n */\nexport function dumpBase64(data: any, options?: MarshalOptions) {\n const marshaller = new Marshaller(options || { version: 2 });\n marshaller.marshal(data);\n return marshaller.dumpAsBuffer().toString(\"base64\");\n}\n\n/**\n * Loads data from a base64 string, as serialized by dumpBase64().\n */\nexport function loadBase64(data: string, options?: UnmarshalOptions) {\n return loads(Buffer.from(data, \"base64\"), options);\n}\n" }, { "path": "app/common/normalizedDateTimeString.ts", "content": "import moment from \"moment-timezone\";\n\n/**\n * Output an ISO8601 format datetime string, with timezone.\n * Any string fed in without timezone is expected to be in UTC.\n *\n * When connected to postgres, dates will be extracted as Date objects,\n * with timezone information. The normalization done here is not\n * really needed in this case.\n *\n * Timestamps in SQLite are stored as UTC, and read as strings\n * (without timezone information). The normalization here is\n * pretty important in this case.\n */\nexport function normalizedDateTimeString(dateTime: any): string {\n if (!dateTime) { return dateTime; }\n if (dateTime instanceof Date) {\n return moment(dateTime).toISOString();\n }\n if (typeof dateTime === \"string\" || typeof dateTime === \"number\") {\n // When SQLite returns a string, it will be in UTC.\n // Need to make sure it actually have timezone info in it\n // (will not by default).\n return moment.utc(dateTime).toISOString();\n }\n throw new Error(`normalizedDateTimeString cannot handle ${dateTime}`);\n}\n" }, { "path": "app/common/orgNameUtils.ts", "content": "const BLACKLISTED_SUBDOMAINS = new Set([\n // from wiki page as of 2018-12-14\n \"aws\",\n \"gristlogin\",\n \"issues\",\n \"metrics\",\n \"phab\",\n \"releases\",\n \"test\",\n \"vpn\",\n \"www\",\n\n // A few more reserved just in case. The minimum length requirement would eliminate\n // some in any case, but specified here also in case that minimum changes.\n \"w\", \"ww\", \"wwww\", \"wwwww\",\n \"docs\", \"api\", \"static\",\n \"ftp\", \"imap\", \"pop\", \"smtp\", \"mail\", \"git\", \"blog\", \"wiki\", \"support\", \"kb\", \"help\",\n \"admin\", \"store\", \"dev\", \"beta\",\n \"community\", \"try\", \"wpx\", \"telemetry\",\n\n // a few random tech brands\n \"google\", \"apple\", \"microsoft\", \"ms\", \"facebook\", \"fb\", \"twitter\", \"youtube\", \"yt\",\n\n // updates for new special domains\n \"current\", \"staging\", \"prod\", \"login\", \"login-dev\", \"login-s\",\n\n // some domains that look suspicious\n \"1ogin\", \"1ogin-dev\", \"1ogin-s\",\n]);\n\n/**\n *\n * Checks whether the subdomain is on the list of forbidden subdomains.\n * See /documentation/urls.md#organization-subdomains\n *\n * Also enforces various sanity checks.\n *\n * Throws if the subdomain is invalid.\n *\n */\nexport function checkSubdomainValidity(subdomain: string): void {\n // stick with limited alphanumeric subdomains.\n if (!(/^[a-z0-9][-a-z0-9]*$/.test(subdomain))) {\n throw new Error(\"Domain must include lower-case letters, numbers, and dashes only.\");\n }\n // 'docs-*' is reserved for personal orgs.\n if (subdomain.startsWith(\"docs-\")) { throw new Error('Domain cannot use reserved prefix \"docs-\".'); }\n // 'o-*' is reserved for automatic org domains.\n if (subdomain.startsWith(\"o-\")) { throw new Error('Domain cannot use reserved prefix \"o-\".'); }\n // 'doc-worker-*' is reserved for doc workers.\n if (subdomain.startsWith(\"doc-worker-\")) { throw new Error('Domain cannot use reserved prefix \"doc-worker-\".'); }\n // special subdomains like _domainkey.\n if (subdomain.startsWith(\"_\")) { throw new Error('Domain cannot use reserved prefix \"_\".'); }\n // some domains are currently in use for testing v1.\n if (subdomain.startsWith(\"v1-\")) { throw new Error('Domain cannot use reserved prefix \"v1-\".'); }\n // check limit of 63 characters on dns label.\n if (subdomain.length > 63) { throw new Error(\"Domain must contain less than 64 characters.\"); }\n // check the subdomain isn't too short.\n if (subdomain.length <= 2) { throw new Error(\"Domain must contain more than 2 characters.\"); }\n // a small blacklist prepared by hand.\n if (BLACKLISTED_SUBDOMAINS.has(subdomain)) { throw new Error(\"Invalid domain value.\"); }\n}\n" }, { "path": "app/common/parseDate.ts", "content": "import { getDistinctValues, isNonNullish } from \"app/common/gutil\";\n\nimport guessFormat from \"@gristlabs/moment-guess/dist/bundle.js\";\nimport escapeRegExp from \"lodash/escapeRegExp\";\nimport last from \"lodash/last\";\nimport memoize from \"lodash/memoize\";\n// Simply importing 'moment-guess' inconsistently imports bundle.js or bundle.esm.js depending on environment\nimport moment from \"moment-timezone\";\n\n// When using YY format, use a consistent interpretation in datepicker and in moment parsing: add\n// 2000 if the result is at most 10 years greater than the current year; otherwise add 1900. See\n// https://bootstrap-datepicker.readthedocs.io/en/latest/options.html#assumenearbyyear and\n// \"Parsing two digit years\" in https://momentjs.com/docs/#/parsing/string-format/.\nexport const TWO_DIGIT_YEAR_THRESHOLD = 10;\nconst MAX_TWO_DIGIT_YEAR = new Date().getFullYear() + TWO_DIGIT_YEAR_THRESHOLD - 2000;\n\n// Moment suggests that overriding this is fine, but we need to force TypeScript to allow it.\n(moment as any).parseTwoDigitYear = function(yearString: string): number {\n const year = parseInt(yearString, 10);\n return year + (year > MAX_TWO_DIGIT_YEAR ? 1900 : 2000);\n};\n\n// Order of formats to try if the date cannot be parsed as the currently set format.\n// Formats are parsed in momentjs strict mode, but separator matching and the MM/DD\n// two digit requirement are ignored. Also, partial completion is permitted, so formats\n// may match even if only beginning elements are provided.\n// TODO: These should be affected by the user's locale/settings.\n// TODO: We may want to consider adding default time formats as well to support more\n// time formats.\nconst PARSER_FORMATS: string[] = [\n \"M D YYYY\",\n \"M D YY\",\n \"M D\",\n \"M\",\n \"MMMM D YYYY\",\n \"MMMM D\",\n \"MMMM Do YYYY\",\n \"MMMM Do\",\n \"D MMMM YYYY\",\n \"D MMMM\",\n \"Do MMMM YYYY\",\n \"Do MMMM\",\n \"MMMM\",\n \"MMM D YYYY\",\n \"MMM D\",\n \"MMM Do YYYY\",\n \"MMM Do\",\n \"D MMM YYYY\",\n \"D MMM\",\n \"Do MMM YYYY\",\n \"Do MMM\",\n \"MMM\",\n \"YYYY M D\",\n \"YYYY M\",\n \"YYYY\",\n \"D M YYYY\",\n \"D M YY\",\n \"D M\",\n \"D\",\n];\n\nconst UNAMBIGUOUS_FORMATS = [\n \"YYYY M D\",\n ...PARSER_FORMATS.filter(f => f.includes(\"MMM\")),\n];\n\nconst TIME_REGEX = /(?:^|\\s+|T)(?:(\\d\\d?)(?::(\\d\\d?)(?::(\\d\\d?))?)?|(\\d\\d?)(\\d\\d))\\s*([ap]m?)?$/i;\n// [^a-zA-Z] because no letters are allowed directly before the abbreviation\nconst UTC_REGEX = /[^a-zA-Z](UTC?|GMT|Z)$/i;\nconst NUMERIC_TZ_REGEX = /([+-]\\d\\d?)(?::?(\\d\\d))?$/i;\n\n// Not picky about separators, so replace them in the date and format strings to be spaces.\nconst SEPARATORS = /[\\W_]+/g;\n\nconst tzAbbreviations = memoize((tzName: string): RegExp => {\n // Some abbreviations are just e.g. +05\n // and escaping the + seems better than filtering\n const abbreviations = new Set(moment.tz.zone(tzName)!.abbrs.map(escapeRegExp));\n\n const union = [...abbreviations].join(\"|\");\n\n // [^a-zA-Z] because no letters are allowed directly before the abbreviation\n // so for example CEST won't match even if EST does\n return new RegExp(`[^a-zA-Z](${union})$`, \"i\");\n});\n\ninterface ParseOptions {\n time?: string;\n dateFormat?: string;\n timeFormat?: string;\n timezone?: string;\n}\n\n/**\n * parseDate - Attempts to parse a date string using several common formats. Returns the\n * timestamp of the parsed date in seconds since epoch, or returns null on failure.\n * @param {String} date - The date string to parse.\n * @param {String} options.dateFormat - The preferred momentjs format to use to parse the\n * date. This is attempted before the default formats.\n * @param {String} options.time - The time string to parse.\n * @param {String} options.timeFormat - The momentjs format to use to parse the time. This\n * must be given if options.time is given.\n * @param {String} options.timezone - The timezone string for the date/time, which affects\n * the resulting timestamp.\n */\nexport function parseDate(date: string, options: ParseOptions = {}): number | null {\n // If no date, return null.\n if (!date) {\n return null;\n }\n\n // If this looks like a timestamp (string with 9 or more digits), just return it.\n const timestamp = parseTimeStamp(date);\n if (timestamp !== null) {\n return timestamp;\n }\n\n const dateFormat = options.dateFormat || \"YYYY-MM-DD\";\n const dateFormats = [..._buildVariations(dateFormat, date), ...PARSER_FORMATS];\n const cleanDate = date.replace(SEPARATORS, \" \");\n let datetime = cleanDate.trim();\n let timeformat = \"\";\n let time = options.time;\n if (time) {\n const parsedTimeZone = parseTimeZone(time, options.timezone!);\n const parsedTime = standardizeTime(parsedTimeZone.remaining);\n if (!parsedTime || parsedTime.remaining) {\n return null;\n }\n time = parsedTime.time;\n const { tzOffset } = parsedTimeZone;\n datetime += \" \" + time + tzOffset;\n timeformat = \" HH:mm:ss\" + (tzOffset ? \"Z\" : \"\");\n }\n for (const format of dateFormats) {\n const fullFormat = format + timeformat;\n const m = moment.tz(datetime, fullFormat, true, options.timezone || \"UTC\");\n if (m.isValid()) {\n return m.unix();\n }\n }\n return null;\n}\n\n/**\n * Similar to parseDate, with these differences:\n * - Only for a date (no time part)\n * - Only falls back to UNAMBIGUOUS_FORMATS, not the full PARSER_FORMATS\n * - Optionally adds all dates which match some format to `results`, otherwise returns first match.\n * This is safer so it can be used for parsing when pasting a large number of dates\n * and won't silently swap around day and month.\n */\nexport function parseDateStrict(\n date: string, dateFormat: string | null, results?: Set, timezone: string = \"UTC\",\n): number | undefined {\n if (!date) {\n return;\n }\n // If this looks like a timestamp (string with 9 or more digits), just return it.\n const timestamp = parseTimeStamp(date);\n if (timestamp !== null) {\n return timestamp;\n }\n dateFormat = dateFormat || \"YYYY-MM-DD\";\n const dateFormats = [..._buildVariations(dateFormat, date), ...UNAMBIGUOUS_FORMATS];\n const cleanDate = date.replace(SEPARATORS, \" \").trim();\n for (const format of dateFormats) {\n const m = moment.tz(cleanDate, format, true, timezone);\n if (m.isValid()) {\n const value = m.valueOf() / 1000;\n if (results) {\n results.add(value);\n } else {\n return value;\n }\n }\n }\n}\n\nexport function parseDateTime(dateTime: string, options: ParseOptions): number | undefined {\n dateTime = dateTime.trim();\n if (!dateTime) {\n return;\n }\n\n const dateFormat = options.dateFormat || \"YYYY-MM-DD\";\n const timezone = options.timezone || \"UTC\";\n\n const dateOnly = parseDateStrict(dateTime, dateFormat, undefined, timezone);\n if (dateOnly) {\n return dateOnly;\n }\n\n const parsedTimeZone = parseTimeZone(dateTime, timezone);\n let tzOffset = \"\";\n if (parsedTimeZone) {\n tzOffset = parsedTimeZone.tzOffset;\n dateTime = parsedTimeZone.remaining;\n }\n\n const parsedTime = standardizeTime(dateTime);\n if (!parsedTime) {\n return;\n }\n\n dateTime = parsedTime.remaining;\n const date = parseDateStrict(dateTime, dateFormat);\n\n if (!date) {\n return;\n }\n\n // date is a timestamp of midnight in UTC, so to get a formatted representation (for parsing\n // together with time), take care to interpret it in UTC.\n const dateString = moment.unix(date).utc().format(\"YYYY-MM-DD\");\n dateTime = dateString + \" \" + parsedTime.time + tzOffset;\n const fullFormat = \"YYYY-MM-DD HH:mm:ss\" + (tzOffset ? \"Z\" : \"\");\n return moment.tz(dateTime, fullFormat, true, timezone).valueOf() / 1000;\n}\n\n// Helper function to get the partial format string based on the input. Momentjs has a feature\n// which allows defaulting to the current year, month and/or day if not accounted for in the\n// parser. We remove any parts of the parser not given in the input to take advantage of this\n// feature.\nfunction _getPartialFormat(input: string, format: string): string {\n // Define a regular expression to match contiguous non-separators.\n const re = /Y+|M+o?|D+o?|[a-zA-Z0-9]+/ig;\n // Count the number of meaningful parts in the input.\n const numInputParts = input.match(re)?.length || 0;\n\n // Count the number of parts in the format string.\n let numFormatParts = format.match(re)?.length || 0;\n\n if (numFormatParts > numInputParts) {\n // Remove year from format first, to default to current year.\n if (/Y+/.test(format)) {\n format = format.replace(/Y+/, \" \").trim();\n numFormatParts -= 1;\n }\n if (numFormatParts > numInputParts) {\n // Remove month from format next.\n format = format.replace(/M+/, \" \").trim();\n }\n }\n return format;\n}\n\n// Moment non-strict mode is considered bad, as it's far too lax. But moment's strict mode is too\n// strict. We want to allow YY|YYYY for either year specifier, as well as M for MMM or MMMM month\n// specifiers. It's silly that we need to create multiple format variations to support this.\nfunction _buildVariations(dateFormat: string, date: string) {\n // Momentjs has an undesirable feature in strict mode where MM and DD\n // matches require two digit numbers. Change MM, DD to M, D.\n let format = dateFormat.replace(/MM+/g, m => (m === \"MM\" ? \"M\" : m))\n .replace(/DD+/g, m => (m === \"DD\" ? \"D\" : m))\n .replace(SEPARATORS, \" \")\n .trim();\n\n // Allow the input date to end with a 4-digit year even if the format doesn't mention the year\n if (\n format.includes(\"M\") &&\n format.includes(\"D\") &&\n !format.includes(\"Y\")\n ) {\n format += \" YYYY\";\n }\n\n format = _getPartialFormat(date, format);\n\n // Consider some alternatives to the preferred format.\n const variations = new Set([format]);\n const otherYear = format.replace(/Y{2,4}/, m => (m === \"YY\" ? \"YYYY\" : (m === \"YYYY\" ? \"YY\" : m)));\n variations.add(otherYear);\n variations.add(format.replace(/MMM+/, \"M\"));\n if (otherYear !== format) {\n variations.add(otherYear.replace(/MMM+/, \"M\"));\n }\n return variations;\n}\n\n// Based on private calculateOffset in moment source code.\nfunction calculateOffset(tzMatch: string[]): string {\n const [, hhOffset, mmOffset] = tzMatch;\n const sign = hhOffset.slice(0, 1);\n return sign + hhOffset.slice(1).padStart(2, \"0\") + \":\" + (mmOffset || \"0\").padStart(2, \"0\");\n}\n\nfunction parseTimeZone(str: string, timezone: string): { remaining: string, tzOffset: string } {\n str = str.trim();\n\n let tzMatch = UTC_REGEX.exec(str);\n let matchStart = 0;\n let tzOffset = \"\";\n if (tzMatch) {\n tzOffset = \"+00:00\";\n matchStart = tzMatch.index + 1; // skip [^a-zA-Z] at regex start\n } else {\n tzMatch = NUMERIC_TZ_REGEX.exec(str);\n if (tzMatch) {\n tzOffset = calculateOffset(tzMatch);\n matchStart = tzMatch.index;\n } else if (timezone) {\n // Abbreviations are simply stripped and ignored, so tzOffset is not set in this case\n tzMatch = tzAbbreviations(timezone).exec(str);\n if (tzMatch) {\n matchStart = tzMatch.index + 1; // skip [^a-zA-Z] at regex start\n }\n }\n }\n\n if (tzMatch) {\n str = str.slice(0, matchStart).trim();\n }\n\n return { remaining: str, tzOffset };\n}\n\n// Parses time of the form, roughly, HH[:MM[:SS]][am|pm]. Returns the time in the\n// standardized HH:mm:ss format.\n// This turns out easier than coaxing moment to parse time sensibly and flexibly.\nfunction standardizeTime(timeString: string): { remaining: string, time: string } | undefined {\n const match = TIME_REGEX.exec(timeString);\n if (!match) {\n return;\n }\n let hours = parseInt(match[1] || match[4], 10);\n const mm = (match[2] || match[5] || \"0\").padStart(2, \"0\");\n const ss = (match[3] || \"0\").padStart(2, \"0\");\n const ampm = (match[6] || \"\").toLowerCase();\n if (hours < 12 && hours > 0 && ampm.startsWith(\"p\")) {\n hours += 12;\n } else if (hours === 12 && ampm.startsWith(\"a\")) {\n hours = 0;\n }\n const hh = String(hours).padStart(2, \"0\");\n return { remaining: timeString.slice(0, match.index).trim(), time: `${hh}:${mm}:${ss}` };\n}\n\n/**\n * Guesses a full date[time] format that best matches the given strings.\n * If several formats match equally well, picks the last one lexicographically to match the old date guessing.\n * This means formats with an early Y and/or M are favoured.\n * If no formats match, returns the default YYYY-MM-DD.\n */\nexport function guessDateFormat(values: (string | null)[], timezone: string = \"UTC\"): string {\n const formats = guessDateFormats(values, timezone);\n if (!formats) {\n return \"YYYY-MM-DD\";\n }\n return last(formats)!;\n}\n\n/**\n * Returns all full date[time] formats that best match the given strings.\n * If several formats match equally well, returns them all.\n * May return null if there are no matching formats or choosing one is too expensive.\n */\nexport function guessDateFormats(values: (string | null)[], timezone: string = \"UTC\"): string[] | null {\n const dateStrings: string[] = values.filter(isNonNullish);\n const sample = getDistinctValues(dateStrings, 100);\n const formats: Record = {};\n for (const dateString of sample) {\n let guessed: string | string[];\n try {\n guessed = guessFormat(dateString);\n } catch {\n continue;\n }\n if (typeof guessed === \"string\") {\n guessed = [guessed];\n }\n for (const guess of guessed) {\n formats[guess] = 0;\n }\n }\n const formatKeys = Object.keys(formats);\n if (!formatKeys.length || formatKeys.length > 10) {\n return null;\n }\n\n for (const format of formatKeys) {\n for (const dateString of dateStrings) {\n const m = moment.tz(dateString, format, true, timezone);\n if (m.isValid()) {\n formats[format] += 1;\n }\n }\n }\n\n const maxCount = Math.max(...Object.values(formats));\n // Return all formats that tied for first place.\n // Sort lexicographically for consistency in tests and with the old dateguess.py.\n return formatKeys.filter(format => formats[format] === maxCount).sort();\n}\n\nexport const dateFormatOptions = [\n \"YYYY-MM-DD\",\n \"MM-DD-YYYY\",\n \"MM/DD/YYYY\",\n \"MM-DD-YY\",\n \"MM/DD/YY\",\n \"DD MMM YYYY\",\n \"MMMM Do, YYYY\",\n \"DD-MM-YYYY\",\n];\n\nexport const timeFormatOptions = [\n \"h:mma\",\n \"h:mma z\",\n \"HH:mm\",\n \"HH:mm z\",\n \"HH:mm:ss\",\n \"HH:mm:ss z\",\n];\n\n/**\n * Construct widget options for a Date or DateTime column based on a single moment string\n * which may or may not contain both date and time parts.\n * If defaultTimeFormat is true, fallback to a non-empty default time format when none is found in fullFormat.\n */\nexport function dateTimeWidgetOptions(fullFormat: string, defaultTimeFormat: boolean) {\n const index = fullFormat.match(/[hHkaAmsSzZT]|$/)!.index!;\n const dateFormat = fullFormat.substr(0, index).trim();\n const timeFormat = fullFormat.substr(index).trim() || (defaultTimeFormat ? timeFormatOptions[0] : \"\");\n return {\n dateFormat,\n timeFormat,\n isCustomDateFormat: !dateFormatOptions.includes(dateFormat),\n isCustomTimeFormat: !timeFormatOptions.includes(timeFormat),\n };\n}\n\n/**\n * Attempts to parse a timestamp string. Returns the timestamp in seconds\n * since epoch, or returns null on failure. Accepts only strings with 9 to 11 digits.\n * Lowest 11 digit timestamp is 2286-11-20, so we don't consider them valid.\n */\nexport function parseTimeStamp(date: string): number | null {\n // If this looks like a timestamp (number with 9 or more digits), just return it.\n // This covers most of the cases leaving some time around the unix epoch not covered.\n // So time before 100 000 000 (1974-04-26) is not covered. Also negative values\n // are also not supported, as they overlap with the YYYYYY date format.\n if (date && /^[1-9]\\d{8,9}$/.test(date)) {\n const parsedDate = moment(date, \"X\");\n if (parsedDate.isValid()) {\n return parsedDate.unix();\n }\n }\n return null;\n}\n" }, { "path": "app/common/plugin.ts", "content": "/**\n * Plugin's utilities common to server and client.\n */\nimport { BarePlugin, Implementation } from \"app/plugin/PluginManifest\";\n\nexport type LocalPluginKind = \"installed\" | \"builtIn\";\n\nexport interface ImplDescription {\n localPluginId: string;\n implementation: Implementation;\n}\n\nexport interface FileParser {\n fileExtensions: string[];\n parseOptions?: ImplDescription;\n fileParser: ImplDescription;\n}\n\n// Deprecated, use FileParser or ImportSource instead.\nexport interface FileImporter {\n id: string;\n fileExtensions?: string[];\n script?: string;\n scriptFullPath?: string;\n filePicker?: string;\n filePickerFullPath?: string;\n}\n\n/**\n * Manifest parsing error.\n */\nexport interface ManifestParsingError {\n yamlError?: any;\n jsonError?: any;\n cannotReadError?: any;\n missingEntryErrors?: string;\n}\n\n/**\n * Whether the importer provides a file picker.\n */\nexport function isPicker(importer: FileImporter): boolean {\n return importer.filePicker !== undefined;\n}\n\n/**\n * A Plugin that was found in the system, either installed or builtin.\n */\nexport interface LocalPlugin {\n /**\n * the plugin's manifest\n */\n manifest: BarePlugin;\n /**\n * The path to the plugin's folder.\n */\n path: string;\n /**\n * A name to uniquely identify a LocalPlugin.\n */\n readonly id: string;\n}\n\nexport interface DirectoryScanEntry {\n manifest?: BarePlugin;\n /**\n * User-friendly error messages.\n */\n errors?: any[];\n path: string;\n id: string;\n}\n\n/**\n * The contributions type.\n */\nexport type Contribution = \"importSource\" | \"fileParser\";\n" }, { "path": "app/common/resetOrg.ts", "content": "import { isOwner } from \"app/common/roles\";\nimport { ManagerDelta, PermissionDelta, UserAPI } from \"app/common/UserAPI\";\n\n/**\n * A utility to reset an organization into the state it would have when first\n * created - no docs, one workspace called \"Home\", a single user. Should be\n * called by a user who is both an owner of the org and a billing manager.\n */\nexport async function resetOrg(api: UserAPI, org: string | number) {\n const session = await api.getSessionActive();\n if (!isOwner(session.org)) {\n throw new Error(\"user must be an owner of the org to be reset\");\n }\n const billing = api.getBillingAPI();\n // If billing api is not available, don't bother setting billing manager.\n const account = await billing.getBillingAccount().catch(e => null);\n if (account && !account.managers.some(manager => (manager.id === session.user.id))) {\n throw new Error(\"user must be a billing manager\");\n }\n const wss = await api.getOrgWorkspaces(org);\n for (const ws of wss) {\n if (!ws.isSupportWorkspace) {\n await api.deleteWorkspace(ws.id);\n }\n }\n await api.newWorkspace({ name: \"Home\" }, org);\n const permissions: PermissionDelta = { users: {} };\n for (const user of (await api.getOrgAccess(org)).users) {\n if (user.id !== session.user.id) {\n permissions.users![user.email] = null;\n }\n }\n await api.updateOrgPermissions(org, permissions);\n // For non-individual accounts, update billing managers (individual accounts will\n // throw an error if we try to do this).\n if (account && !account.individual) {\n const managers: ManagerDelta = { users: {} };\n for (const user of account.managers) {\n if (user.id !== session.user.id) {\n managers.users[user.email] = null;\n }\n }\n await billing.updateBillingManagers(managers);\n }\n return api;\n}\n" }, { "path": "app/common/roles.ts", "content": "import { Organization } from \"app/common/UserAPI\";\n\nexport const OWNER = \"owners\";\nexport const EDITOR = \"editors\";\nexport const VIEWER = \"viewers\";\nexport const GUEST = \"guests\";\nexport const MEMBER = \"members\";\n\n// Roles ordered from most to least permissive.\nconst roleOrder: (Role | null)[] = [OWNER, EDITOR, VIEWER, MEMBER, GUEST, null];\n\nexport type BasicRole = \"owners\" | \"editors\" | \"viewers\";\nexport type NonMemberRole = BasicRole | \"guests\";\nexport type NonGuestRole = BasicRole | \"members\";\nexport type Role = NonMemberRole | \"members\";\n\n// Returns the BasicRole (or null) with the same effective access as the given role.\nexport function getEffectiveRole(role: Role | null): BasicRole | null {\n if (role === GUEST || role === MEMBER) {\n return VIEWER;\n } else {\n return role;\n }\n}\n\nexport function canEditAccess(role: string | null): boolean {\n return role === OWNER;\n}\n\n// Note that while canEdit has the same return value as canDelete, the functions are\n// kept separate as they may diverge in the future.\nexport function canEdit(role: string | null): boolean {\n return role === OWNER || role === EDITOR;\n}\n\nexport function canDelete(role: string | null): boolean {\n return role === OWNER || role === EDITOR;\n}\n\nexport function canView(role: string | null): boolean {\n return role !== null;\n}\n\nexport function isOwner(resource: { access: Role | null } | null): resource is { access: Role } {\n return resource?.access === OWNER;\n}\n\nexport function isOwnerOrEditor(resource: { access: Role | null } | null): resource is { access: Role } {\n return canEdit(resource?.access ?? null);\n}\n\nexport function canUpgradeOrg(org: Organization | null): org is Organization {\n // TODO: Need to consider billing managers and support user.\n return isOwner(org);\n}\n\n// Returns true if the role string is a valid role or null.\nexport function isValidRole(role: string | null): role is Role | null {\n return (roleOrder as (string | null)[]).includes(role);\n}\n\n// Returns true if the role string is a valid non-Guest, non-Member, non-null role.\nexport function isBasicRole(role: string | null): role is BasicRole {\n return Boolean(role && role !== GUEST && role !== MEMBER && isValidRole(role));\n}\n\n// Returns true if the role string is a valid non-Guest, non-null role.\nexport function isNonGuestRole(role: string | null): role is NonGuestRole {\n return Boolean(role && role !== GUEST && isValidRole(role));\n}\n\n/**\n * Returns out of any number of group role names the one that offers more permissions. The function\n * is overloaded so that the output type matches the specificity of the input values.\n */\nexport function getStrongestRole(...args: T[]): T {\n return getFirstMatchingRole(roleOrder, args);\n}\n\n/**\n * Returns out of any number of group role names the one that offers fewer permissions. The function\n * is overloaded so that the output type matches the specificity of the input values.\n */\nexport function getWeakestRole(...args: T[]): T {\n return getFirstMatchingRole(roleOrder.slice().reverse(), args);\n}\n\n// Returns which of the `anyOf` args comes first in `array`. Helper for getStrongestRole\n// and getWeakestRole.\nfunction getFirstMatchingRole(array: (Role | null)[], anyOf: T[]): T {\n if (anyOf.length === 0) {\n throw new Error(`getFirstMatchingRole: No roles given`);\n }\n for (const role of anyOf) {\n if (!isValidRole(role)) {\n throw new Error(`getFirstMatchingRole: Invalid role ${role}`);\n }\n }\n return array.find(item => anyOf.includes(item as T)) as T;\n}\n" }, { "path": "app/common/schema.ts", "content": "/* eslint-disable */\n\n/*** THIS FILE IS AUTO-GENERATED BY core/sandbox/gen_js_schema.py ***/\n\nimport { GristObjCode } from \"app/plugin/GristData\";\n\n// tslint:disable:object-literal-key-quotes\n\nexport const SCHEMA_VERSION = 46;\n\nexport const schema = {\n\n \"_grist_DocInfo\": {\n docId : \"Text\",\n peers : \"Text\",\n basketId : \"Text\",\n schemaVersion : \"Int\",\n timezone : \"Text\",\n documentSettings : \"Text\",\n },\n\n \"_grist_Tables\": {\n tableId : \"Text\",\n primaryViewId : \"Ref:_grist_Views\",\n summarySourceTable : \"Ref:_grist_Tables\",\n onDemand : \"Bool\",\n rawViewSectionRef : \"Ref:_grist_Views_section\",\n recordCardViewSectionRef: \"Ref:_grist_Views_section\",\n },\n\n \"_grist_Tables_column\": {\n parentId : \"Ref:_grist_Tables\",\n parentPos : \"PositionNumber\",\n colId : \"Text\",\n type : \"Text\",\n widgetOptions : \"Text\",\n isFormula : \"Bool\",\n formula : \"Text\",\n label : \"Text\",\n description : \"Text\",\n untieColIdFromLabel : \"Bool\",\n summarySourceCol : \"Ref:_grist_Tables_column\",\n displayCol : \"Ref:_grist_Tables_column\",\n visibleCol : \"Ref:_grist_Tables_column\",\n rules : \"RefList:_grist_Tables_column\",\n reverseCol : \"Ref:_grist_Tables_column\",\n recalcWhen : \"Int\",\n recalcDeps : \"RefList:_grist_Tables_column\",\n },\n\n \"_grist_Imports\": {\n tableRef : \"Ref:_grist_Tables\",\n origFileName : \"Text\",\n parseFormula : \"Text\",\n delimiter : \"Text\",\n doublequote : \"Bool\",\n escapechar : \"Text\",\n quotechar : \"Text\",\n skipinitialspace : \"Bool\",\n encoding : \"Text\",\n hasHeaders : \"Bool\",\n },\n\n \"_grist_External_database\": {\n host : \"Text\",\n port : \"Int\",\n username : \"Text\",\n dialect : \"Text\",\n database : \"Text\",\n storage : \"Text\",\n },\n\n \"_grist_External_table\": {\n tableRef : \"Ref:_grist_Tables\",\n databaseRef : \"Ref:_grist_External_database\",\n tableName : \"Text\",\n },\n\n \"_grist_TableViews\": {\n tableRef : \"Ref:_grist_Tables\",\n viewRef : \"Ref:_grist_Views\",\n },\n\n \"_grist_TabItems\": {\n tableRef : \"Ref:_grist_Tables\",\n viewRef : \"Ref:_grist_Views\",\n },\n\n \"_grist_TabBar\": {\n viewRef : \"Ref:_grist_Views\",\n tabPos : \"PositionNumber\",\n },\n\n \"_grist_Pages\": {\n viewRef : \"Ref:_grist_Views\",\n indentation : \"Int\",\n pagePos : \"PositionNumber\",\n shareRef : \"Ref:_grist_Shares\",\n options : \"Text\",\n },\n\n \"_grist_Views\": {\n name : \"Text\",\n type : \"Text\",\n layoutSpec : \"Text\",\n },\n\n \"_grist_Views_section\": {\n tableRef : \"Ref:_grist_Tables\",\n parentId : \"Ref:_grist_Views\",\n parentKey : \"Text\",\n title : \"Text\",\n description : \"Text\",\n defaultWidth : \"Int\",\n borderWidth : \"Int\",\n theme : \"Text\",\n options : \"Text\",\n chartType : \"Text\",\n layoutSpec : \"Text\",\n filterSpec : \"Text\",\n sortColRefs : \"Text\",\n linkSrcSectionRef : \"Ref:_grist_Views_section\",\n linkSrcColRef : \"Ref:_grist_Tables_column\",\n linkTargetColRef : \"Ref:_grist_Tables_column\",\n embedId : \"Text\",\n rules : \"RefList:_grist_Tables_column\",\n shareOptions : \"Text\",\n },\n\n \"_grist_Views_section_field\": {\n parentId : \"Ref:_grist_Views_section\",\n parentPos : \"PositionNumber\",\n colRef : \"Ref:_grist_Tables_column\",\n width : \"Int\",\n widgetOptions : \"Text\",\n displayCol : \"Ref:_grist_Tables_column\",\n visibleCol : \"Ref:_grist_Tables_column\",\n filter : \"Text\",\n rules : \"RefList:_grist_Tables_column\",\n },\n\n \"_grist_Validations\": {\n formula : \"Text\",\n name : \"Text\",\n tableRef : \"Int\",\n },\n\n \"_grist_REPL_Hist\": {\n code : \"Text\",\n outputText : \"Text\",\n errorText : \"Text\",\n },\n\n \"_grist_Attachments\": {\n fileIdent : \"Text\",\n fileName : \"Text\",\n fileType : \"Text\",\n fileSize : \"Int\",\n fileExt : \"Text\",\n imageHeight : \"Int\",\n imageWidth : \"Int\",\n timeDeleted : \"DateTime\",\n timeUploaded : \"DateTime\",\n },\n\n \"_grist_Triggers\": {\n tableRef : \"Ref:_grist_Tables\",\n eventTypes : \"ChoiceList\",\n isReadyColRef : \"Ref:_grist_Tables_column\",\n actions : \"Text\",\n label : \"Text\",\n memo : \"Text\",\n enabled : \"Bool\",\n watchedColRefList : \"RefList:_grist_Tables_column\",\n options : \"Text\",\n condition : \"Text\",\n },\n\n \"_grist_ACLRules\": {\n resource : \"Ref:_grist_ACLResources\",\n permissions : \"Int\",\n principals : \"Text\",\n aclFormula : \"Text\",\n aclColumn : \"Ref:_grist_Tables_column\",\n aclFormulaParsed : \"Text\",\n permissionsText : \"Text\",\n rulePos : \"PositionNumber\",\n userAttributes : \"Text\",\n memo : \"Text\",\n },\n\n \"_grist_ACLResources\": {\n tableId : \"Text\",\n colIds : \"Text\",\n },\n\n \"_grist_ACLPrincipals\": {\n type : \"Text\",\n userEmail : \"Text\",\n userName : \"Text\",\n groupName : \"Text\",\n instanceId : \"Text\",\n },\n\n \"_grist_ACLMemberships\": {\n parent : \"Ref:_grist_ACLPrincipals\",\n child : \"Ref:_grist_ACLPrincipals\",\n },\n\n \"_grist_Filters\": {\n viewSectionRef : \"Ref:_grist_Views_section\",\n colRef : \"Ref:_grist_Tables_column\",\n filter : \"Text\",\n pinned : \"Bool\",\n },\n\n \"_grist_Cells\": {\n tableRef : \"Ref:_grist_Tables\",\n colRef : \"Ref:_grist_Tables_column\",\n rowId : \"Int\",\n root : \"Bool\",\n parentId : \"Ref:_grist_Cells\",\n type : \"Int\",\n content : \"Text\",\n userRef : \"Text\",\n timeCreated : \"DateTime\",\n timeUpdated : \"DateTime\",\n resolved : \"Bool\",\n },\n\n \"_grist_Shares\": {\n linkId : \"Text\",\n options : \"Text\",\n label : \"Text\",\n description : \"Text\",\n },\n\n};\n\nexport interface SchemaTypes {\n\n \"_grist_DocInfo\": {\n docId: string;\n peers: string;\n basketId: string;\n schemaVersion: number;\n timezone: string;\n documentSettings: string;\n };\n\n \"_grist_Tables\": {\n tableId: string;\n primaryViewId: number;\n summarySourceTable: number;\n onDemand: boolean;\n rawViewSectionRef: number;\n recordCardViewSectionRef: number;\n };\n\n \"_grist_Tables_column\": {\n parentId: number;\n parentPos: number;\n colId: string;\n type: string;\n widgetOptions: string;\n isFormula: boolean;\n formula: string;\n label: string;\n description: string;\n untieColIdFromLabel: boolean;\n summarySourceCol: number;\n displayCol: number;\n visibleCol: number;\n rules: [GristObjCode.List, ...number[]]|null;\n reverseCol: number;\n recalcWhen: number;\n recalcDeps: [GristObjCode.List, ...number[]]|null;\n };\n\n \"_grist_Imports\": {\n tableRef: number;\n origFileName: string;\n parseFormula: string;\n delimiter: string;\n doublequote: boolean;\n escapechar: string;\n quotechar: string;\n skipinitialspace: boolean;\n encoding: string;\n hasHeaders: boolean;\n };\n\n \"_grist_External_database\": {\n host: string;\n port: number;\n username: string;\n dialect: string;\n database: string;\n storage: string;\n };\n\n \"_grist_External_table\": {\n tableRef: number;\n databaseRef: number;\n tableName: string;\n };\n\n \"_grist_TableViews\": {\n tableRef: number;\n viewRef: number;\n };\n\n \"_grist_TabItems\": {\n tableRef: number;\n viewRef: number;\n };\n\n \"_grist_TabBar\": {\n viewRef: number;\n tabPos: number;\n };\n\n \"_grist_Pages\": {\n viewRef: number;\n indentation: number;\n pagePos: number;\n shareRef: number;\n options: string;\n };\n\n \"_grist_Views\": {\n name: string;\n type: string;\n layoutSpec: string;\n };\n\n \"_grist_Views_section\": {\n tableRef: number;\n parentId: number;\n parentKey: string;\n title: string;\n description: string;\n defaultWidth: number;\n borderWidth: number;\n theme: string;\n options: string;\n chartType: string;\n layoutSpec: string;\n filterSpec: string;\n sortColRefs: string;\n linkSrcSectionRef: number;\n linkSrcColRef: number;\n linkTargetColRef: number;\n embedId: string;\n rules: [GristObjCode.List, ...number[]]|null;\n shareOptions: string;\n };\n\n \"_grist_Views_section_field\": {\n parentId: number;\n parentPos: number;\n colRef: number;\n width: number;\n widgetOptions: string;\n displayCol: number;\n visibleCol: number;\n filter: string;\n rules: [GristObjCode.List, ...number[]]|null;\n };\n\n \"_grist_Validations\": {\n formula: string;\n name: string;\n tableRef: number;\n };\n\n \"_grist_REPL_Hist\": {\n code: string;\n outputText: string;\n errorText: string;\n };\n\n \"_grist_Attachments\": {\n fileIdent: string;\n fileName: string;\n fileType: string;\n fileSize: number;\n fileExt: string;\n imageHeight: number;\n imageWidth: number;\n timeDeleted: number;\n timeUploaded: number;\n };\n\n \"_grist_Triggers\": {\n tableRef: number;\n eventTypes: [GristObjCode.List, ...string[]]|null;\n isReadyColRef: number;\n actions: string;\n label: string;\n memo: string;\n enabled: boolean;\n watchedColRefList: [GristObjCode.List, ...number[]]|null;\n options: string;\n condition: string;\n };\n\n \"_grist_ACLRules\": {\n resource: number;\n permissions: number;\n principals: string;\n aclFormula: string;\n aclColumn: number;\n aclFormulaParsed: string;\n permissionsText: string;\n rulePos: number;\n userAttributes: string;\n memo: string;\n };\n\n \"_grist_ACLResources\": {\n tableId: string;\n colIds: string;\n };\n\n \"_grist_ACLPrincipals\": {\n type: string;\n userEmail: string;\n userName: string;\n groupName: string;\n instanceId: string;\n };\n\n \"_grist_ACLMemberships\": {\n parent: number;\n child: number;\n };\n\n \"_grist_Filters\": {\n viewSectionRef: number;\n colRef: number;\n filter: string;\n pinned: boolean;\n };\n\n \"_grist_Cells\": {\n tableRef: number;\n colRef: number;\n rowId: number;\n root: boolean;\n parentId: number;\n type: number;\n content: string;\n userRef: string;\n timeCreated: number;\n timeUpdated: number;\n resolved: boolean;\n };\n\n \"_grist_Shares\": {\n linkId: string;\n options: string;\n label: string;\n description: string;\n };\n\n}\n" }, { "path": "app/common/tagManager.ts", "content": "/**\n * Returns the Google Tag Manager snippet to insert into of the page, if\n * `tagId` is set to a non-empty value. Otherwise returns an empty string.\n */\nexport function getTagManagerSnippet(tagId?: string) {\n // Note also that we only insert the snippet for the . The second recommended part (for\n // ) is for